From 98bc5f9af1400f7a90d13cffcc3a543fb42cb7d5 Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Sun, 15 Nov 2020 20:18:54 +0100 Subject: [PATCH 01/14] db: collect information of files in the system --- .gitignore | 1 + Makefile | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..50c72fb --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/internals-*.db diff --git a/Makefile b/Makefile index 08e9342..ef9915f 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,36 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf +DB = internals-$(shell sw_vers -productVersion).db +DB_TARGETS = db_files -.PHONY: all +.PHONY: all $(DB_TARGETS) -all: internals.txt +all: internals.txt $(DB) ifneq ($(wildcard $(MY_INTERNALS)),) internals.txt: $(MY_INTERNALS) textutil -cat txt "$<" -output $@ endif + +$(DB): + @$(MAKE) --silent --jobs=1 $(DB_TARGETS) | sqlite3 -bail $@ + +db_files: + if ! csrutil status | grep -Fq disabled ; then \ + printf '\033[1mdisable SIP to get complete file information\033[m\n' >&2 ; \ + echo 'FAIL;' ; \ + exit 1 ; \ + fi + printf '\033[1mcollecting file information...\033[m\n' >&2 + echo 'DROP TABLE IF EXISTS files;' + echo 'CREATE TABLE files (id INTEGER PRIMARY KEY, os TEXT, path TEXT, executable BOOLEAN);' + sudo find /Library /System /bin /dev /private /sbin /usr ! \( -path /System/Volumes/Data -prune \) 2> /dev/null | \ + sed "s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('macOS', '&');/" + find $(HOME)/Library | \ + sed "s|^$(HOME)|~|;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('macOS', '&');/" + cd /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot ; find . | \ + sed "1d;s/\\.//;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('iOS', '&');/" + cd /Applications/Xcode.app/Contents/Developer/Platforms/AppleTVOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/tvOS.simruntime/Contents/Resources/RuntimeRoot ; find . | \ + sed "1d;s/\\.//;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('tvOS', '&');/" + cd /Applications/Xcode.app/Contents/Developer/Platforms/WatchOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/watchOS.simruntime/Contents/Resources/RuntimeRoot ; find . | \ + sed "1d;s/\\.//;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('watchOS', '&');/" + echo 'CREATE INDEX files_path ON files (path);' From 91f62823cd20992aa2e48bd106dd796f0dc7dcf9 Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Mon, 23 Nov 2020 16:12:06 +0100 Subject: [PATCH 02/14] db: refactor to separate iterator function --- Makefile | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index ef9915f..51e6129 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,27 @@ endif $(DB): @$(MAKE) --silent --jobs=1 $(DB_TARGETS) | sqlite3 -bail $@ + +# MARK: - data extraction helpers + +prefix = $$(case $(1) in \ + (macOS) ;; \ + (iOS) echo /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot ;; \ + (tvOS) echo /Applications/Xcode.app/Contents/Developer/Platforms/AppleTVOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/tvOS.simruntime/Contents/Resources/RuntimeRoot ;; \ + (watchOS) echo /Applications/Xcode.app/Contents/Developer/Platforms/WatchOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/watchOS.simruntime/Contents/Resources/RuntimeRoot ;; \ + esac) + +find = \ + { \ + $(2) find /Library /System /bin /dev /private /sbin /usr ! \( -path /System/Volumes/Data -prune \) $(1) 2> /dev/null | sed 's/^/macOS /' ; \ + cd $(call prefix,iOS) ; find . $(1) | sed '1d;s/^\./iOS /' ; \ + cd $(call prefix,tvOS) ; find . $(1) | sed '1d;s/^\./tvOS /' ; \ + cd $(call prefix,watchOS) ; find . $(1) | sed '1d;s/^\./watchOS /' ; \ + } + + +# MARK: - generator targets for database + db_files: if ! csrutil status | grep -Fq disabled ; then \ printf '\033[1mdisable SIP to get complete file information\033[m\n' >&2 ; \ @@ -23,14 +44,6 @@ db_files: printf '\033[1mcollecting file information...\033[m\n' >&2 echo 'DROP TABLE IF EXISTS files;' echo 'CREATE TABLE files (id INTEGER PRIMARY KEY, os TEXT, path TEXT, executable BOOLEAN);' - sudo find /Library /System /bin /dev /private /sbin /usr ! \( -path /System/Volumes/Data -prune \) 2> /dev/null | \ - sed "s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('macOS', '&');/" - find $(HOME)/Library | \ - sed "s|^$(HOME)|~|;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('macOS', '&');/" - cd /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot ; find . | \ - sed "1d;s/\\.//;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('iOS', '&');/" - cd /Applications/Xcode.app/Contents/Developer/Platforms/AppleTVOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/tvOS.simruntime/Contents/Resources/RuntimeRoot ; find . | \ - sed "1d;s/\\.//;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('tvOS', '&');/" - cd /Applications/Xcode.app/Contents/Developer/Platforms/WatchOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/watchOS.simruntime/Contents/Resources/RuntimeRoot ; find . | \ - sed "1d;s/\\.//;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('watchOS', '&');/" + $(call find,,sudo) | sed -E "s/'/''/g;s/([^ ]*) (.*)/INSERT INTO files (os, path) VALUES('\1', '\2');/" + find $(HOME)/Library | sed "s|^$(HOME)|~|;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('macOS', '&');/" echo 'CREATE INDEX files_path ON files (path);' From caa1b1dce32410d6debd713d161baf90db1da984 Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 14:29:44 +0100 Subject: [PATCH 03/14] db: store compressed database file automatically decompress before using --- .gitignore | 2 +- Makefile | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 50c72fb..d87447d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -/internals-*.db +/internals-*.db.lz diff --git a/Makefile b/Makefile index 51e6129..bfc535c 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,29 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf -DB = internals-$(shell sw_vers -productVersion).db +DB := $(if $(DB),$(DB:.lz=),internals-$(shell sw_vers -productVersion).db) DB_TARGETS = db_files .PHONY: all $(DB_TARGETS) +.INTERMEDIATE: $(DB) -all: internals.txt $(DB) +all: internals.txt $(DB).lz ifneq ($(wildcard $(MY_INTERNALS)),) internals.txt: $(MY_INTERNALS) textutil -cat txt "$<" -output $@ endif +ifneq ($(wildcard $(DB).lz),) +$(DB): $(DB).lz + compression_tool -decode -i $< -o $@ +else $(DB): @$(MAKE) --silent --jobs=1 $(DB_TARGETS) | sqlite3 -bail $@ +$(DB).lz: $(DB) + compression_tool -encode -i $< -o $@ + tmutil addexclusion $@ +endif + # MARK: - data extraction helpers From 76c9417244566f465c2dd9ea15caf327f20626af Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Sat, 21 Nov 2020 19:37:02 +0100 Subject: [PATCH 04/14] db: use transactions for better database performance --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bfc535c..86fb2de 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,10 @@ find = \ # MARK: - generator targets for database -db_files: +$(DB_TARGETS):: + echo 'BEGIN IMMEDIATE TRANSACTION;' + +db_files:: if ! csrutil status | grep -Fq disabled ; then \ printf '\033[1mdisable SIP to get complete file information\033[m\n' >&2 ; \ echo 'FAIL;' ; \ @@ -57,3 +60,6 @@ db_files: $(call find,,sudo) | sed -E "s/'/''/g;s/([^ ]*) (.*)/INSERT INTO files (os, path) VALUES('\1', '\2');/" find $(HOME)/Library | sed "s|^$(HOME)|~|;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('macOS', '&');/" echo 'CREATE INDEX files_path ON files (path);' + +$(DB_TARGETS):: + echo 'COMMIT TRANSACTION;' From d85af9cea803509a41f4eaf2f831ff8edc0f3dcb Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 14:28:33 +0100 Subject: [PATCH 05/14] check: existence of mentioned files first check also sets up checks infrastructure --- Makefile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 86fb2de..d0f9d24 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,12 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf DB := $(if $(DB),$(DB:.lz=),internals-$(shell sw_vers -productVersion).db) DB_TARGETS = db_files +CHECK_TARGETS = check_files -.PHONY: all $(DB_TARGETS) +.PHONY: all check $(DB_TARGETS) $(CHECK_TARGETS) .INTERMEDIATE: $(DB) -all: internals.txt $(DB).lz +all: $(DB).lz check ifneq ($(wildcard $(MY_INTERNALS)),) internals.txt: $(MY_INTERNALS) @@ -24,6 +25,10 @@ $(DB).lz: $(DB) tmutil addexclusion $@ endif +check: internals.txt + @LANG=en sort --ignore-case $< | diff -uw $< - + @$(MAKE) --silent --jobs=1 $(CHECK_TARGETS) + # MARK: - data extraction helpers @@ -63,3 +68,12 @@ db_files:: $(DB_TARGETS):: echo 'COMMIT TRANSACTION;' + + +# MARK: - check targets for internals.txt + +check_files: internals.txt $(DB) + printf '\033[1mchecking files...\033[m\n' >&2 + grep -ow '~\?/[^,;]*' $< | sed -E 's/ \(.*\)$$//;s/^\/(etc|var)\//\/private&/' | \ + sed "s/'/''/g;s|.*|SELECT count(*), '&' FROM files WHERE path GLOB '&';|" | \ + sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" From 6853a9111d1f5274f1fb29022a30daeba150d8a6 Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 14:26:17 +0100 Subject: [PATCH 06/14] db: collect information about binaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • library linkage • entitlements • strings in the binary --- Makefile | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d0f9d24..491d536 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf DB := $(if $(DB),$(DB:.lz=),internals-$(shell sw_vers -productVersion).db) -DB_TARGETS = db_files +DB_TARGETS = db_files db_binaries CHECK_TARGETS = check_files .PHONY: all check $(DB_TARGETS) $(CHECK_TARGETS) @@ -47,6 +47,8 @@ find = \ cd $(call prefix,watchOS) ; find . $(1) | sed '1d;s/^\./watchOS /' ; \ } +file = SELECT id, $(1) FROM files WHERE os = '$$os' AND path = '$$(echo "$$path" | sed "s/'/''/g")' + # MARK: - generator targets for database @@ -66,6 +68,27 @@ db_files:: find $(HOME)/Library | sed "s|^$(HOME)|~|;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('macOS', '&');/" echo 'CREATE INDEX files_path ON files (path);' +db_binaries:: + printf '\033[1mcollecting executable information...\033[m\n' >&2 + echo 'DROP TABLE IF EXISTS linkages;' + echo 'DROP TABLE IF EXISTS entitlements;' + echo 'DROP TABLE IF EXISTS strings;' + echo 'CREATE TABLE linkages (id INTEGER REFERENCES files, dylib TEXT);' + echo 'CREATE TABLE entitlements (id INTEGER REFERENCES files, plist JSON);' + echo 'CREATE TABLE strings (id INTEGER REFERENCES files, string TEXT);' + $(call find,-follow -type f -perm +111) | while read -r os path ; do \ + echo "UPDATE files SET executable = true WHERE os = '$$os' AND path = '$$path';" ; \ + if test -r "$(call prefix,$$os)$$path" && file --no-dereference --brief --mime-type "$(call prefix,$$os)$$path" | grep -Fq application/x-mach-binary ; then \ + objdump --macho --dylibs-used "$(call prefix,$$os)$$path" | \ + sed "1d;s/^.//;s/ ([^)]*)$$//;s/'/''/g;s|.*|INSERT INTO linkages $(call file,'&');|" ; \ + codesign --display --entitlements - "$(call prefix,$$os)$$path" 2> /dev/null | \ + sed 1d | plutil -convert json - -o - | \ + sed "/^: Property List error/d;/^{}/d;s/'/''/g;s|.*|INSERT INTO entitlements $(call file,json('&'));\n|" ; \ + strings -n 8 "$(call prefix,$$os)$$path" | \ + LANG=C sed "s/'/''/g;s|.*|INSERT INTO strings $(call file,'&');|" ; \ + fi ; \ + done + $(DB_TARGETS):: echo 'COMMIT TRANSACTION;' From ee5938c46f45c0a61931e2bb3ebd32178843eaa7 Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Fri, 20 Nov 2020 21:46:27 +0100 Subject: [PATCH 07/14] db: include developer tools in scan --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 491d536..d5a54d3 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,7 @@ prefix = $$(case $(1) in \ find = \ { \ $(2) find /Library /System /bin /dev /private /sbin /usr ! \( -path /System/Volumes/Data -prune \) $(1) 2> /dev/null | sed 's/^/macOS /' ; \ + cd /Applications/Xcode.app/Contents/Developer ; find Library Toolchains Tools usr $(1) | sed 's|^|macOS /Applications/Xcode.app/Contents/Developer/|' ; \ cd $(call prefix,iOS) ; find . $(1) | sed '1d;s/^\./iOS /' ; \ cd $(call prefix,tvOS) ; find . $(1) | sed '1d;s/^\./tvOS /' ; \ cd $(call prefix,watchOS) ; find . $(1) | sed '1d;s/^\./watchOS /' ; \ From 051dd0f167beb471727bf58912949454d123709c Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 10:08:15 +0100 Subject: [PATCH 08/14] db: extract dylibs from dyld cache and scan uses dyld_shared_cache_util tool from the Nix flake --- Makefile | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d5a54d3..0975608 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ $(DB): $(DB).lz: $(DB) compression_tool -encode -i $< -o $@ tmutil addexclusion $@ + rm -rf dyld endif check: internals.txt @@ -32,8 +33,18 @@ check: internals.txt # MARK: - data extraction helpers +NIX = $(shell nix-build --no-out-link -A nixFlakes '')/bin/nix +DSCU = $(shell \ + $(NIX) --experimental-features 'nix-command flakes' build --no-write-lock-file .\#dyld-shared-cache && \ + readlink result && rm result)/bin/dyld_shared_cache_util + +dyld: /System/Library/dyld/dyld_shared_cache_$(shell uname -m) + $(DSCU) -extract $@ $< + find $@ -type f -print0 | xargs -0 chmod a+x + prefix = $$(case $(1) in \ (macOS) ;; \ + (macOS-dyld) echo $(dir $(realpath $(firstword $(MAKEFILE_LIST))))/dyld ;; \ (iOS) echo /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot ;; \ (tvOS) echo /Applications/Xcode.app/Contents/Developer/Platforms/AppleTVOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/tvOS.simruntime/Contents/Resources/RuntimeRoot ;; \ (watchOS) echo /Applications/Xcode.app/Contents/Developer/Platforms/WatchOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/watchOS.simruntime/Contents/Resources/RuntimeRoot ;; \ @@ -43,6 +54,7 @@ find = \ { \ $(2) find /Library /System /bin /dev /private /sbin /usr ! \( -path /System/Volumes/Data -prune \) $(1) 2> /dev/null | sed 's/^/macOS /' ; \ cd /Applications/Xcode.app/Contents/Developer ; find Library Toolchains Tools usr $(1) | sed 's|^|macOS /Applications/Xcode.app/Contents/Developer/|' ; \ + test -d "$(call prefix,macOS-dyld)" && cd "$(call prefix,macOS-dyld)" && find . $(1) | sed '1d;s/^\./macOS-dyld /' ; \ cd $(call prefix,iOS) ; find . $(1) | sed '1d;s/^\./iOS /' ; \ cd $(call prefix,tvOS) ; find . $(1) | sed '1d;s/^\./tvOS /' ; \ cd $(call prefix,watchOS) ; find . $(1) | sed '1d;s/^\./watchOS /' ; \ @@ -56,7 +68,7 @@ file = SELECT id, $(1) FROM files WHERE os = '$$os' AND path = '$$(echo "$$path" $(DB_TARGETS):: echo 'BEGIN IMMEDIATE TRANSACTION;' -db_files:: +db_files:: dyld if ! csrutil status | grep -Fq disabled ; then \ printf '\033[1mdisable SIP to get complete file information\033[m\n' >&2 ; \ echo 'FAIL;' ; \ @@ -69,7 +81,7 @@ db_files:: find $(HOME)/Library | sed "s|^$(HOME)|~|;s/'/''/g;s/.*/INSERT INTO files (os, path) VALUES('macOS', '&');/" echo 'CREATE INDEX files_path ON files (path);' -db_binaries:: +db_binaries:: dyld printf '\033[1mcollecting executable information...\033[m\n' >&2 echo 'DROP TABLE IF EXISTS linkages;' echo 'DROP TABLE IF EXISTS entitlements;' From e03a6cd0ca5e77aca0e29ea55773324bf0a4fbf0 Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 10:08:44 +0100 Subject: [PATCH 09/14] check: executables and frameworks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • check command line tools • check framework names • check server names as strings in binaries --- Makefile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0975608..119e571 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf DB := $(if $(DB),$(DB:.lz=),internals-$(shell sw_vers -productVersion).db) DB_TARGETS = db_files db_binaries -CHECK_TARGETS = check_files +CHECK_TARGETS = check_files check_binaries .PHONY: all check $(DB_TARGETS) $(CHECK_TARGETS) .INTERMEDIATE: $(DB) @@ -113,3 +113,17 @@ check_files: internals.txt $(DB) grep -ow '~\?/[^,;]*' $< | sed -E 's/ \(.*\)$$//;s/^\/(etc|var)\//\/private&/' | \ sed "s/'/''/g;s|.*|SELECT count(*), '&' FROM files WHERE path GLOB '&';|" | \ sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" + +check_binaries: internals.txt $(DB) + printf '\033[1mchecking command line tools...\033[m\n' >&2 + grep -o 'command line tools\?: [^;]*' $< | sed 's/^[^:]*: //;s/ //g;s/([^)]*)//g' | tr , '\n' | \ + sed "s/'/''/g;s|.*|SELECT count(*), '&' FROM files WHERE executable = true AND path GLOB '*/&';|" | \ + sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" + printf '\033[1mchecking frameworks...\033[m\n' >&2 + grep -ow '[[:alnum:]]*\.framework[[:alnum:]/.]*' $< | \ + sed "s|/|/*/|g;s/'/''/g;s|.*|SELECT count(*), '&' FROM files WHERE executable = true AND path GLOB '*/&/*';|" | \ + sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" + printf '\033[1mchecking servers...\033[m\n' >&2 + grep -o 'servers\?: [^;]*' $< | sed 's/^[^:]*: //;s/ //g;s/([^)]*)//g' | tr , '\n' | \ + sed "s/'/''/g;s/.*/SELECT count(*), '&' FROM strings WHERE string GLOB '*&*';/" | \ + sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" From 5c7181b1f92ada4e762e3d667fce024edef759bb Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 10:09:01 +0100 Subject: [PATCH 10/14] db: collect contents of asset catalogs --- Makefile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 119e571..9e782dc 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf DB := $(if $(DB),$(DB:.lz=),internals-$(shell sw_vers -productVersion).db) -DB_TARGETS = db_files db_binaries +DB_TARGETS = db_files db_binaries db_assets CHECK_TARGETS = check_files check_binaries .PHONY: all check $(DB_TARGETS) $(CHECK_TARGETS) @@ -34,6 +34,9 @@ check: internals.txt # MARK: - data extraction helpers NIX = $(shell nix-build --no-out-link -A nixFlakes '')/bin/nix +ACEXTRACT = $(shell \ + $(NIX) --experimental-features 'nix-command flakes' build --no-write-lock-file .\#acextract && \ + readlink result && rm result)/bin/acextract DSCU = $(shell \ $(NIX) --experimental-features 'nix-command flakes' build --no-write-lock-file .\#dyld-shared-cache && \ readlink result && rm result)/bin/dyld_shared_cache_util @@ -102,6 +105,15 @@ db_binaries:: dyld fi ; \ done +db_assets:: + printf '\033[1mcollecting asset catalog information...\033[m\n' >&2 + echo 'DROP TABLE IF EXISTS assets;' + echo 'CREATE TABLE assets (id INTEGER REFERENCES files, name TEXT);' + $(call find,-type f -name '*.car') | while read -r os path ; do \ + test -r "$(call prefix,$$os)$$path" && $(ACEXTRACT) --list --input "$(call prefix,$$os)$$path" | \ + sed "1d;s/'/''/g;s|.*|INSERT INTO assets $(call file,'&');|" ; \ + done + $(DB_TARGETS):: echo 'COMMIT TRANSACTION;' From e1278f9a21d5465cf4ee85ab8956ed71d22d8150 Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 10:09:18 +0100 Subject: [PATCH 11/14] db: collect launchd service information --- Makefile | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9e782dc..2d8e290 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf DB := $(if $(DB),$(DB:.lz=),internals-$(shell sw_vers -productVersion).db) -DB_TARGETS = db_files db_binaries db_assets +DB_TARGETS = db_files db_binaries db_assets db_services CHECK_TARGETS = check_files check_binaries .PHONY: all check $(DB_TARGETS) $(CHECK_TARGETS) @@ -64,6 +64,7 @@ find = \ } file = SELECT id, $(1) FROM files WHERE os = '$$os' AND path = '$$(echo "$$path" | sed "s/'/''/g")' +, = , # for entering a literal comma as part of a function argument # MARK: - generator targets for database @@ -114,6 +115,16 @@ db_assets:: sed "1d;s/'/''/g;s|.*|INSERT INTO assets $(call file,'&');|" ; \ done +db_services:: + printf '\033[1mcollecting launchd service information...\033[m\n' >&2 + echo 'DROP TABLE IF EXISTS services;' + echo 'CREATE TABLE services (id INTEGER REFERENCES files, kind TEXT, plist JSON);' + $(call find,-type f -name '*.plist' -path '*/LaunchAgents/*' -o -path '*/LaunchDaemons/*') | while read -r os path ; do \ + case "$$path" in (*/LaunchAgents/*) kind=agent ;; (*/LaunchDaemons/*) kind=daemon ;; esac ; \ + test -r "$(call prefix,$$os)$$path" && plutil -convert json "$(call prefix,$$os)$$path" -o - | \ + sed "s/'/''/g;s|.*|INSERT INTO services $(call file,'$$kind'$(,)json('&'));\n|" ; \ + done + $(DB_TARGETS):: echo 'COMMIT TRANSACTION;' From b5bcc70bd6c67bf3092f5e2476ac5899e8bbc1d2 Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 10:09:31 +0100 Subject: [PATCH 12/14] check: launchd services MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • check listed service labels • check host/task special ports --- Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2d8e290..6cbdd89 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf DB := $(if $(DB),$(DB:.lz=),internals-$(shell sw_vers -productVersion).db) DB_TARGETS = db_files db_binaries db_assets db_services -CHECK_TARGETS = check_files check_binaries +CHECK_TARGETS = check_files check_binaries check_services .PHONY: all check $(DB_TARGETS) $(CHECK_TARGETS) .INTERMEDIATE: $(DB) @@ -150,3 +150,13 @@ check_binaries: internals.txt $(DB) grep -o 'servers\?: [^;]*' $< | sed 's/^[^:]*: //;s/ //g;s/([^)]*)//g' | tr , '\n' | \ sed "s/'/''/g;s/.*/SELECT count(*), '&' FROM strings WHERE string GLOB '*&*';/" | \ sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" + +check_services: internals.txt $(DB) + printf '\033[1mchecking launchd services...\033[m\n' >&2 + grep -o 'launchd services\?: [^;]*' $< | sed 's/^[^:]*: //;s/ //g;s/([^)]*)//g' | tr , '\n' | \ + sed "s/'/''/g;s|.*|SELECT count(*), '&' FROM services, json_each(plist) WHERE key = 'Label' AND value = '&';|" | \ + sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" + printf '\033[1mchecking special ports...\033[m\n' >&2 + grep -o '[^ ]* special port [0-9]*' $< | \ + sed -E "s/'/''/g;s/(host|task) special port ([0-9]+)/SELECT count(*), '&' FROM services, json_tree(plist, '$$.MachServices') WHERE key LIKE '\1SpecialPort' AND value = \2;/" | \ + sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" From 17478b01446de711dae13a7b7f25d57a14cbdd1c Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 14:33:10 +0100 Subject: [PATCH 13/14] db: collect Info.plist files --- Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6cbdd89..4774b25 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf DB := $(if $(DB),$(DB:.lz=),internals-$(shell sw_vers -productVersion).db) -DB_TARGETS = db_files db_binaries db_assets db_services +DB_TARGETS = db_files db_binaries db_manifests db_assets db_services CHECK_TARGETS = check_files check_binaries check_services .PHONY: all check $(DB_TARGETS) $(CHECK_TARGETS) @@ -106,6 +106,15 @@ db_binaries:: dyld fi ; \ done +db_manifests:: + printf '\033[1mcollecting Info.plist information...\033[m\n' >&2 + echo 'DROP TABLE IF EXISTS info;' + echo 'CREATE TABLE info (id INTEGER REFERENCES files, plist JSON);' + $(call find,-type f -name 'Info.plist') | while read -r os path ; do \ + test -r "$(call prefix,$$os)$$path" && plutil -convert json "$(call prefix,$$os)$$path" -o - | \ + sed "/: invalid object/d;s/'/''/g;s|.*|INSERT INTO info $(call file,json('&'));\n|" ; \ + done + db_assets:: printf '\033[1mcollecting asset catalog information...\033[m\n' >&2 echo 'DROP TABLE IF EXISTS assets;' From 7a892360c8e238b8fd24a826f0d7e9c80f2c012b Mon Sep 17 00:00:00 2001 From: Michael Roitzsch Date: Tue, 24 Nov 2020 14:33:25 +0100 Subject: [PATCH 14/14] check: extension points listed extension points should be valid NSExtensionPointIdentifiers --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4774b25..61a71bf 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ MY_INTERNALS = $(HOME)/Library/Mobile\ Documents/com~apple~TextEdit/Documents/Apple\ Internals.rtf DB := $(if $(DB),$(DB:.lz=),internals-$(shell sw_vers -productVersion).db) DB_TARGETS = db_files db_binaries db_manifests db_assets db_services -CHECK_TARGETS = check_files check_binaries check_services +CHECK_TARGETS = check_files check_binaries check_manifests check_services .PHONY: all check $(DB_TARGETS) $(CHECK_TARGETS) .INTERMEDIATE: $(DB) @@ -160,6 +160,12 @@ check_binaries: internals.txt $(DB) sed "s/'/''/g;s/.*/SELECT count(*), '&' FROM strings WHERE string GLOB '*&*';/" | \ sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" +check_manifests: internals.txt $(DB) + printf '\033[1mchecking extension points...\033[m\n' >&2 + grep -o 'extension points\?: [^;]*' $< | sed 's/^[^:]*: //;s/ //g;s/([^)]*)//g' | tr , '\n' | \ + sed "s/'/''/g;s|.*|SELECT count(*), '&' FROM info, json_each(plist, '$$.NSExtension') WHERE key = 'NSExtensionPointIdentifier' AND value = '&';|" | \ + sqlite3 $(DB) | sed -n "/^0|/{s/^0|//;p;}" + check_services: internals.txt $(DB) printf '\033[1mchecking launchd services...\033[m\n' >&2 grep -o 'launchd services\?: [^;]*' $< | sed 's/^[^:]*: //;s/ //g;s/([^)]*)//g' | tr , '\n' | \