diff --git a/toolkit/Makefile b/toolkit/Makefile index 2b6f430a9..e13a2771e 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -1,55 +1,15 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -# Contains: -# - Definitions -# - High Level Targets -# - Submake Includes - -######## DEFINITIONS ######## - toolkit_root := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) SCRIPTS_DIR ?= $(toolkit_root)/scripts -######## SET REMAINING FLAGS ######## - -RUN_CHECK ?= n - -##help:var:REPO_SNAPSHOT_TIME:=Posix time to be used as a snapshot for remote repositories when fetching packages. Example: REPO_SNAPSHOT_TIME="1724119509". -CACHE_DIR ?= -PACKAGE_CACHE_SUMMARY ?= -IMAGE_CACHE_SUMMARY ?= -INITRD_CACHE_SUMMARY ?= -PACKAGE_ARCHIVE ?= -PACKAGE_BUILD_RETRIES ?= 0 -CHECK_BUILD_RETRIES ?= 0 -EXTRA_BUILD_LAYERS ?= 0 -REFRESH_WORKER_CHROOT ?= y -TARGET_ARCH ?= -ALLOW_TOOLCHAIN_REBUILDS ?= n -CACHED_PACKAGES_ARCHIVE ?= -USE_CCACHE ?= n - -# Folder defines TOOLS_DIR ?= $(toolkit_root)/tools TOOL_BINS_DIR ?= $(toolkit_root)/out/tools -RESOURCES_DIR ?= $(toolkit_root)/resources PROJECT_ROOT ?= $(realpath $(toolkit_root)/..) BUILD_DIR ?= $(PROJECT_ROOT)/build OUT_DIR ?= $(PROJECT_ROOT)/out -CCACHE_DIR ?= $(PROJECT_ROOT)/ccache -CCACHE_CONFIG ?= $(RESOURCES_DIR)/manifests/package/ccache-configuration.json - -# Sub-folder defines -LOGS_DIR ?= $(BUILD_DIR)/logs -CACHED_RPMS_DIR ?= $(BUILD_DIR)/rpm_cache -STATUS_FLAGS_DIR ?= $(BUILD_DIR)/make_status -CHROOT_DIR ?= $(BUILD_DIR)/worker/chroot -IMAGEGEN_DIR ?= $(BUILD_DIR)/imagegen -PROFILE_DIR ?= $(BUILD_DIR)/profile - -IMAGES_DIR ?= $(OUT_DIR)/images ######## COMMON MAKEFILE UTILITIES ######## @@ -66,13 +26,11 @@ LOG_LEVEL ?= info # always,auto,never LOG_COLOR ?= auto STOP_ON_WARNING ?= n -STOP_ON_PKG_FAIL ?= n -STOP_ON_FETCH_FAIL ?= n ######## HIGH LEVEL TARGETS ######## .PHONY: all clean -all: toolchain go-tools chroot-tools +all: go-tools ######## SUBMAKE INCLUDES ######## @@ -88,10 +46,6 @@ include $(SCRIPTS_DIR)/build_tag_imagecustomizer.mk # go-tools, clean-go-tools, go-tidy-all (tidy go utilities before committing) go-test-coverage include $(SCRIPTS_DIR)/tools.mk -# Create images with: -# image, iso, clean-imagegen -include $(SCRIPTS_DIR)/imggen.mk - ##help:target:clean=Clean all built files. # Each component provides specific clean implementations which clean depends on. # They are guaranteed to run first and will verify there are no existing mount points diff --git a/toolkit/scripts/imggen.mk b/toolkit/scripts/imggen.mk deleted file mode 100644 index 7887b00a0..000000000 --- a/toolkit/scripts/imggen.mk +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -$(call create_folder,$(IMAGEGEN_DIR)) - -# Resources -config_name = $(notdir $(CONFIG_FILE:%.json=%)) -config_other_files = $(if $(CONFIG_FILE),$(call shell_real_build_only, $(SCRIPTS_DIR)/get_config_deps.sh $(CONFIG_FILE)),) -assets_dir = $(RESOURCES_DIR)/assets/ -assets_files = $(call shell_real_build_only, find $(assets_dir)) -imggen_local_repo = $(MANIFESTS_DIR)/image/local.repo -imagefetcher_local_repo = $(MANIFESTS_DIR)/package/local.repo -imagefetcher_cloned_repo = $(MANIFESTS_DIR)/package/fetcher.repo -ifeq ($(build_arch),aarch64) -initrd_config_json = $(RESOURCES_DIR)/imageconfigs/iso_initrd_arm64.json -else -initrd_config_json = $(RESOURCES_DIR)/imageconfigs/iso_initrd.json -endif -initrd_assets_dir = $(RESOURCES_DIR)/imageconfigs/additionalfiles/iso_initrd/ -initrd_scripts_dir = $(RESOURCES_DIR)/imageconfigs/postinstallscripts/iso_initrd/ -ifeq ($(build_arch),aarch64) -initrd_packages_json = $(RESOURCES_DIR)/imageconfigs/packagelists/iso-initrd-packages-arm64.json -else -initrd_packages_json = $(RESOURCES_DIR)/imageconfigs/packagelists/iso-initrd-packages.json -endif -initrd_packages_json += $(RESOURCES_DIR)/imageconfigs/packagelists/accessibility-packages.json -initrd_assets_files = $(initrd_packages_json) $(call shell_real_build_only, find $(initrd_assets_dir) $(initrd_scripts_dir)) -meta_user_data_files = $(META_USER_DATA_DIR)/user-data $(META_USER_DATA_DIR)/meta-data -ova_ovfinfo = $(assets_dir)/ova/ovfinfo.txt -ova_vmxtemplate = $(assets_dir)/ova/vmx-template - -# Built RPMs -imggen_rpms = $(call shell_real_build_only, find $(RPMS_DIR) -type f -name '*.rpm') - -# Imagegen workspace and cache -imggen_config_dir = $(IMAGEGEN_DIR)/$(config_name) -workspace_dir = $(imggen_config_dir)/workspace -local_and_external_rpm_cache = $(imggen_config_dir)/package_repo -external_rpm_cache = $(imggen_config_dir)/external_package_repo -image_fetcher_tmp_dir = $(imggen_config_dir)/fetcher_tmp -image_roaster_tmp_dir = $(imggen_config_dir)/roaster_tmp -validate-config = $(STATUS_FLAGS_DIR)/validate-image-config-$(config_name).flag -meta_user_data_tmp_dir = $(IMAGEGEN_DIR)/meta-user-data_tmp -image_package_cache_summary = $(imggen_config_dir)/image_deps.json -image_external_package_cache_summary = $(imggen_config_dir)/image_external_deps.json -image_package_manifest = $(imggen_config_dir)/image_pkg_manifest.json -license_results_file_img = $(imggen_config_dir)/license_check_results.json - -# Outputs -artifact_dir = $(IMAGES_DIR)/$(config_name) -imager_disk_output_dir = $(imggen_config_dir)/imager_output -imager_disk_output_files = $(call shell_real_build_only, find $(imager_disk_output_dir) -not -name '*:*' -not -name '* *') -ifeq ($(build_arch),aarch64) -initrd_img = $(IMAGES_DIR)/iso_initrd_arm64/iso-initrd.img -else -initrd_img = $(IMAGES_DIR)/iso_initrd/iso-initrd.img -endif -meta_user_data_iso = $(IMAGES_DIR)/meta-user-data.iso - -$(call create_folder,$(workspace_dir)) -$(call create_folder,$(imager_disk_output_dir)) -$(call create_folder,$(artifact_dir)) -$(call create_folder,$(meta_user_data_tmp_dir)) - -.PHONY: fetch-image-packages fetch-external-image-packages make-raw-image image iso installer-initrd validate-image-config clean-imagegen - -clean: clean-imagegen -clean-imagegen: - rm -rf $(STATUS_FLAGS_DIR)/build_srpms.flag - rm -rf $(STATUS_FLAGS_DIR)/imager_disk_output.flag - rm -rf $(STATUS_FLAGS_DIR)/validate-image-config-* - rm -rf $(artifact_dir) - rm -rf $(IMAGES_DIR) - @echo Verifying no mountpoints present in $(IMAGEGEN_DIR) - $(SCRIPTS_DIR)/safeunmount.sh "$(IMAGEGEN_DIR)" && \ - rm -rf $(IMAGEGEN_DIR) - -##help:target:fetch-image-packages=Locate and download all packages required for an image build. -fetch-image-packages: $(image_package_cache_summary) - -##help:target:fetch-external-image-packages=Download all external packages required for an image build. -fetch-external-image-packages: $(image_external_package_cache_summary) - -##help:target:validate-image-config=Validate the selected image config. -# Validate the selected config file if any changes occur in the image config base directory. -# Changes to files located outside the base directory will not be detected. -validate-image-config: $(validate-config) -$(STATUS_FLAGS_DIR)/validate-image-config%.flag: $(go-imageconfigvalidator) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(config_other_files) - $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) - $(go-imageconfigvalidator) \ - --input=$(CONFIG_FILE) \ - --dir=$(CONFIG_BASE_DIR) \ - --cpu-prof-file=$(PROFILE_DIR)/imageconfigvalidator.cpu.pprof \ - --mem-prof-file=$(PROFILE_DIR)/imageconfigvalidator.mem.pprof \ - --trace-file=$(PROFILE_DIR)/imageconfigvalidator.trace \ - $(if $(filter y,$(ENABLE_CPU_PROFILE)),--enable-cpu-prof) \ - $(if $(filter y,$(ENABLE_MEM_PROFILE)),--enable-mem-prof) \ - $(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \ - --timestamp-file=$(TIMESTAMP_DIR)/imageconfigvalidator.jsonl && \ - touch $@ - - -imagepkgfetcher_extra_flags := -ifeq ($(DISABLE_UPSTREAM_REPOS),y) -imagepkgfetcher_extra_flags += --disable-upstream-repos -endif - -ifeq ($(DISABLE_DEFAULT_REPOS),y) -imagepkgfetcher_extra_flags += --disable-default-repos -endif - -ifeq ($(USE_PREVIEW_REPO),y) -imagepkgfetcher_extra_flags += --use-preview-repo -endif - -ifneq ($(REPO_SNAPSHOT_TIME),) -imagepkgfetcher_extra_flags += --repo-snapshot-time=$(REPO_SNAPSHOT_TIME) -endif - -$(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(toolchain_rpms) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(RPMS_DIR) $(imggen_rpms) $(depend_REPO_SNAPSHOT_TIME) $(STATUS_FLAGS_DIR)/imagegen_cleanup.flag - $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) - $(go-imagepkgfetcher) \ - --input=$(CONFIG_FILE) \ - --base-dir=$(CONFIG_BASE_DIR) \ - --log-level=$(LOG_LEVEL) \ - --log-file=$(LOGS_DIR)/imggen/imagepkgfetcher.log \ - --log-color=$(LOG_COLOR) \ - --rpm-dir=$(RPMS_DIR) \ - --tmp-dir=$(image_fetcher_tmp_dir) \ - --toolchain-rpms-dir="$(TOOLCHAIN_RPMS_DIR)" \ - --tdnf-worker=$(chroot_worker) \ - --tls-cert=$(TLS_CERT) \ - --tls-key=$(TLS_KEY) \ - $(foreach repo, $(imagefetcher_local_repo) $(imagefetcher_cloned_repo) $(REPO_LIST),--repo-file="$(repo)" ) \ - $(imagepkgfetcher_extra_flags) \ - --input-summary-file=$(IMAGE_CACHE_SUMMARY) \ - --output-summary-file=$@ \ - --output-dir=$(local_and_external_rpm_cache) \ - --cpu-prof-file=$(PROFILE_DIR)/imagepkgfetcher.cpu.pprof \ - --mem-prof-file=$(PROFILE_DIR)/imagepkgfetcher.mem.pprof \ - --trace-file=$(PROFILE_DIR)/imagepkgfetcher.trace \ - $(if $(filter y,$(ENABLE_CPU_PROFILE)),--enable-cpu-prof) \ - $(if $(filter y,$(ENABLE_MEM_PROFILE)),--enable-mem-prof) \ - $(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \ - --timestamp-file=$(TIMESTAMP_DIR)/imagepkgfetcher.jsonl - -##help:target:make-raw-image=Create the raw base image. -make-raw-image: $(imager_disk_output_dir) -$(imager_disk_output_dir): $(STATUS_FLAGS_DIR)/imager_disk_output.flag - @touch $@ - @echo Finished updating $@ - -$(STATUS_FLAGS_DIR)/imager_disk_output.flag: $(go-imager) $(image_package_cache_summary) $(license_results_file_img) $(imggen_local_repo) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(assets_files) $(depend_REPO_SNAPSHOT_TIME) - $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) - mkdir -p $(imager_disk_output_dir) && \ - rm -rf $(imager_disk_output_dir)/* && \ - $(go-imager) \ - --build-dir $(workspace_dir) \ - --input $(CONFIG_FILE) \ - --base-dir=$(CONFIG_BASE_DIR) \ - --log-level=$(LOG_LEVEL) \ - --log-file=$(LOGS_DIR)/imggen/imager.log \ - --log-color=$(LOG_COLOR) \ - --local-repo $(local_and_external_rpm_cache) \ - --tdnf-worker $(chroot_worker) \ - --repo-snapshot-time=$(REPO_SNAPSHOT_TIME) \ - --repo-file=$(imggen_local_repo) \ - --output-image-contents=$(image_package_manifest) \ - --assets $(assets_dir) \ - --output-dir $(imager_disk_output_dir) \ - --cpu-prof-file=$(PROFILE_DIR)/imager.cpu.pprof \ - --mem-prof-file=$(PROFILE_DIR)/imager.mem.pprof \ - --trace-file=$(PROFILE_DIR)/imager.trace \ - $(if $(filter y,$(ENABLE_CPU_PROFILE)),--enable-cpu-prof) \ - $(if $(filter y,$(ENABLE_MEM_PROFILE)),--enable-mem-prof) \ - $(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \ - --timestamp-file=$(TIMESTAMP_DIR)/imager.jsonl \ - --build-number=$(BUILD_ID) && \ - touch $@ - -# Sometimes files will have been deleted, that is fine so long as we were able to detect the change -$(imager_disk_output_dir)/%: ; - -##help:target:image=Generate an image. -image: $(imager_disk_output_dir) $(imager_disk_output_files) $(go-roast) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) - $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) - VMXTEMPLATE=$(ova_vmxtemplate) OVFINFO=$(ova_ovfinfo) \ - $(go-roast) \ - --dir=$(imager_disk_output_dir) \ - --config $(CONFIG_FILE) \ - --output-dir $(artifact_dir) \ - --tmp-dir $(image_roaster_tmp_dir) \ - --release-version $(RELEASE_VERSION) \ - --log-level=$(LOG_LEVEL) \ - --log-file=$(LOGS_DIR)/imggen/roast.log \ - --log-color=$(LOG_COLOR) \ - --image-tag=$(IMAGE_TAG) \ - --cpu-prof-file=$(PROFILE_DIR)/roast.cpu.pprof \ - --mem-prof-file=$(PROFILE_DIR)/roast.mem.pprof \ - --trace-file=$(PROFILE_DIR)/roast.trace \ - $(if $(filter y,$(ENABLE_CPU_PROFILE)),--enable-cpu-prof) \ - $(if $(filter y,$(ENABLE_MEM_PROFILE)),--enable-mem-prof) \ - $(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \ - --timestamp-file=$(TIMESTAMP_DIR)/roast.jsonl - -$(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(chroot_worker) $(graph_file) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(depend_REPO_SNAPSHOT_TIME) $(STATUS_FLAGS_DIR)/imagegen_cleanup.flag - $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) - $(go-imagepkgfetcher) \ - --input=$(CONFIG_FILE) \ - --base-dir=$(CONFIG_BASE_DIR) \ - --log-level=$(LOG_LEVEL) \ - --log-file=$(LOGS_DIR)/imggen/externalimagepkgfetcher.log \ - --log-color=$(LOG_COLOR) \ - --rpm-dir=$(RPMS_DIR) \ - --tmp-dir=$(image_fetcher_tmp_dir) \ - --toolchain-rpms-dir="$(TOOLCHAIN_RPMS_DIR)" \ - --tdnf-worker=$(chroot_worker) \ - --external-only \ - --package-graph=$(graph_file) \ - --tls-cert=$(TLS_CERT) \ - --tls-key=$(TLS_KEY) \ - $(foreach repo, $(imagefetcher_local_repo) $(imagefetcher_cloned_repo) $(REPO_LIST),--repo-file="$(repo)" ) \ - $(imagepkgfetcher_extra_flags) \ - --input-summary-file=$(IMAGE_CACHE_SUMMARY) \ - --output-summary-file=$@ \ - --output-dir=$(external_rpm_cache) \ - --cpu-prof-file=$(PROFILE_DIR)/imagepkgfetcher.cpu.pprof \ - --mem-prof-file=$(PROFILE_DIR)/imagepkgfetcher.mem.pprof \ - --trace-file=$(PROFILE_DIR)/imagepkgfetcher.trace \ - $(if $(filter y,$(ENABLE_CPU_PROFILE)),--enable-cpu-prof) \ - $(if $(filter y,$(ENABLE_MEM_PROFILE)),--enable-mem-prof) \ - $(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \ - --timestamp-file=$(TIMESTAMP_DIR)/imagepkgfetcher.jsonl - -# We need to ensure that initrd_img recursive build will never run concurrently with another build component, so add all ISO prereqs as -# order-only-prerequisites to initrd_img -iso_deps = $(go-isomaker) $(go-imager) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(image_package_cache_summary) $(license_results_file_img) $(depend_REPO_SNAPSHOT_TIME) -# The initrd bundles these files into the image, we should rebuild it if they change -initrd_bundled_files = $(go-liveinstaller) $(go-imager) $(assets_files) $(initrd_assets_files) $(imggen_local_repo) - -$(initrd_img): $(initrd_bundled_files) $(initrd_config_json) $(INITRD_CACHE_SUMMARY) | $(iso_deps) - # Recursive make call to build the initrd image $(artifact_dir)/iso-initrd.img - $(MAKE) image MAKEOVERRIDES= CONFIG_FILE=$(initrd_config_json) IMAGE_CACHE_SUMMARY=$(INITRD_CACHE_SUMMARY) IMAGE_TAG= RELEASE_VERSION=$(RELEASE_VERSION) BUILD_NUMBER=$(BUILD_NUMBER) - -##help:target:installer-initrd=Create the initrd for the ISO installer. -installer-initrd: $(initrd_img) - -##help:target:iso=Create an installable ISO. -iso: $(initrd_img) $(iso_deps) - $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) - $(go-isomaker) \ - --base-dir $(CONFIG_BASE_DIR) \ - --build-dir $(workspace_dir) \ - --initrd-path $(initrd_img) \ - --input $(CONFIG_FILE) \ - --release-version $(RELEASE_VERSION) \ - --resources $(RESOURCES_DIR) \ - --repo-snapshot-time=$(REPO_SNAPSHOT_TIME) \ - --iso-repo $(local_and_external_rpm_cache) \ - --log-level=$(LOG_LEVEL) \ - --log-file=$(LOGS_DIR)/imggen/isomaker.log \ - --log-color=$(LOG_COLOR) \ - $(if $(filter y,$(UNATTENDED_INSTALLER)),--unattended-install) \ - --output-dir $(artifact_dir) \ - --image-tag=$(IMAGE_TAG) - -##help:target:meta-user-data=Create a `meta-user-data.iso` file under `IMAGES_DIR` using `meta-data` and `user-data` from `META_USER_DATA_DIR`. -meta-user-data: $(meta_user_data_files) - cp -t $(meta_user_data_tmp_dir) $(meta_user_data_files) - if [ -n "$(SSH_KEY_FILE)" ]; then \ - sed -i "s|ssh-rsa |`cat $(SSH_KEY_FILE)`|" $(meta_user_data_tmp_dir)/user-data; \ - fi - if [ -n "$(TLS_CERT)" ] && [ -n "$(TLS_KEY)" ] && [ -n "$(CA_CERT)" ]; then \ - $(SCRIPTS_DIR)/addcerts.sh $(meta_user_data_tmp_dir)/user-data $(TLS_CERT) $(TLS_KEY) $(CA_CERT); \ - fi - genisoimage -output $(meta_user_data_iso) -volid cidata -joliet -rock $(meta_user_data_tmp_dir)/* diff --git a/toolkit/scripts/utils.mk b/toolkit/scripts/utils.mk index 4cc53c4f8..3ffb48c0e 100644 --- a/toolkit/scripts/utils.mk +++ b/toolkit/scripts/utils.mk @@ -54,12 +54,6 @@ endef ######## VARIABLE DEPENDENCY TRACKING ######## -# List of variables to watch for changes. -watch_vars=PACKAGE_BUILD_LIST PACKAGE_REBUILD_LIST PACKAGE_IGNORE_LIST REPO_LIST CONFIG_FILE STOP_ON_PKG_FAIL TOOLCHAIN_ARCHIVE REBUILD_TOOLCHAIN SRPM_PACK_LIST SPECS_DIR MAX_CASCADING_REBUILDS RUN_CHECK TEST_RUN_LIST TEST_RERUN_LIST TEST_IGNORE_LIST EXTRA_BUILD_LAYERS LICENSE_CHECK_MODE VALIDATE_TOOLCHAIN_GPG REPO_SNAPSHOT_TIME -# Current list: $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_REPO_LIST) $(depend_CONFIG_FILE) $(depend_STOP_ON_PKG_FAIL) -# $(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) $(depend_SRPM_PACK_LIST) $(depend_SPECS_DIR) $(depend_EXTRA_BUILD_LAYERS) $(depend_MAX_CASCADING_REBUILDS) $(depend_RUN_CHECK) $(depend_TEST_RUN_LIST) -# $(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) $(depend_LICENSE_CHECK_MODE) $(depend_VALIDATE_TOOLCHAIN_GPG) $(depend_REPO_SNAPSHOT_TIME) - .PHONY: variable_depends_on_phony clean-variable_depends_on_phony setfacl_always_run_phony clean: clean-variable_depends_on_phony @@ -90,9 +84,6 @@ $(STATUS_FLAGS_DIR)/$1_tracking_flag: variable_depends_on_phony fi endef -# Invoke the above rule for each tracked variable -$(foreach var,$(watch_vars),$(eval $(call depend_on_var,$(var)))) - # Host's ACLs influence the default permissions of the # files inside the built RPMs. Disabling them for the repository. #