[2025-07-07 13:43:28,746][ INFO][PID:2418953] Marking build as starting [2025-07-07 13:43:28,842][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:43:28,850][ INFO][PID:2418953] VM allocation process starts [2025-07-07 13:43:28,905][ INFO][PID:2418953] Trying to allocate VM: ResallocHost, ticket_id=3817860, requested_tags=['copr_builder', 'arch_aarch64'] [2025-07-07 13:43:31,959][ INFO][PID:2418953] Allocated host ResallocHost, ticket_id=3817860, hostname=18.209.100.254, name=aws_aarch64_spot_prod_05173625_20250707_131227, requested_tags=['copr_builder', 'arch_aarch64'] [2025-07-07 13:43:31,960][ INFO][PID:2418953] Allocating ssh connection to builder [2025-07-07 13:43:31,961][ INFO][PID:2418953] Checking that builder machine is OK [2025-07-07 13:43:32,346][ INFO][PID:2418953] Running remote command: copr-builder-ready fedora-rawhide-aarch64 [2025-07-07 13:43:32,572][ INFO][PID:2418953] Red Hat subscription not needed for fedora-rawhide-aarch64 Builder is ready to be used [2025-07-07 13:43:32,572][ INFO][PID:2418953] Filling build.info file with builder info [2025-07-07 13:43:32,573][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:43:32,574][ INFO][PID:2418953] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "packit", "project_name": "osbuild-osbuild-composer-4764", "project_dirname": "osbuild-osbuild-composer-4764", "submitter": "packit", "ended_on": null, "started_on": 1751895812.5741112, "submitted_on": null, "status": 3, "chroot": "fedora-rawhide-aarch64", "arch": "aarch64", "buildroot_pkgs": [], "task_id": "9245360-fedora-rawhide-aarch64", "build_id": 9245360, "package_name": "osbuild-composer", "package_version": "101-1.20250707134236630338.pr4764.2.gc7e6058f2", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/packit/osbuild-osbuild-composer-4764/osbuild-composer", "git_hash": "f7f9fc3b958b3edcb1de704b96af9c4acfa8112e", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "packit/osbuild-osbuild-composer-4764--packit", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/packit/osbuild-osbuild-composer-4764/fedora-rawhide-aarch64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_aarch64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/packit/osbuild-osbuild-composer-4764", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/packit/osbuild-osbuild-composer-4764", "result_dir": "09245360-osbuild-composer", "built_packages": "", "id": 9245360, "mockchain_macros": { "copr_username": "packit", "copr_projectname": "osbuild-osbuild-composer-4764", "vendor": "Fedora Project COPR (packit/osbuild-osbuild-composer-4764)" } } ] } [2025-07-07 13:43:32,620][ INFO][PID:2418953] Sending fedora-messaging bus message in build.start [2025-07-07 13:43:33,460][ INFO][PID:2418953] Sending fedora-messaging bus message in chroot.start [2025-07-07 13:43:33,493][ INFO][PID:2418953] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9245360-fedora-rawhide-aarch64 --chroot fedora-rawhide-aarch64 --detached [2025-07-07 13:43:33,931][ INFO][PID:2418953] Downloading the builder-live.log file, attempt 1 [2025-07-07 13:43:33,932][ INFO][PID:2418953] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@18.209.100.254 copr-rpmbuild-log [2025-07-07 13:43:38,940][ INFO][PID:2418953] Periodic builder liveness probe: alive [2025-07-07 13:43:38,942][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:43:43,943][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:43:48,945][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:43:53,946][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:43:58,946][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:03,947][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:08,951][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:13,952][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:18,954][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:23,955][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:28,957][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:33,960][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:38,961][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:43,962][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:48,964][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:53,965][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:44:58,966][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:03,967][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:08,969][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:13,970][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:18,971][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:23,973][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:28,974][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:33,976][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:38,977][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:43,978][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:48,979][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:53,980][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:45:58,981][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:03,983][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:08,984][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:13,985][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:18,987][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:23,988][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:28,989][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:33,991][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:38,992][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:43,993][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:48,994][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:53,997][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:46:58,998][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:03,999][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:09,000][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:14,002][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:19,004][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:24,005][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:29,006][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:34,008][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:39,009][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:44,010][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:49,011][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:54,012][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:47:59,013][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:04,014][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:09,015][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:10,313][ ERROR][PID:2418953] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-07 13:48:14,017][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:19,018][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:20,314][WARNING][PID:2418953] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-07 13:48:24,019][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:29,020][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:30,315][ INFO][PID:2418953] Downloading the builder-live.log file, attempt 2 [2025-07-07 13:48:30,318][ INFO][PID:2418953] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@18.209.100.254 copr-rpmbuild-log [2025-07-07 13:48:34,021][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:39,022][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:44,024][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:49,025][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:54,026][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:48:59,027][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:04,028][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:09,029][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:14,030][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:19,031][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:24,033][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:29,034][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:30,386][ ERROR][PID:2418953] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-07 13:49:34,035][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:39,036][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:40,387][WARNING][PID:2418953] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-07 13:49:44,037][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:49,038][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:50,388][ INFO][PID:2418953] Downloading the builder-live.log file, attempt 3 [2025-07-07 13:49:50,389][ INFO][PID:2418953] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@18.209.100.254 copr-rpmbuild-log [2025-07-07 13:49:54,039][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:49:59,041][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:04,042][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:09,043][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:14,044][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:19,046][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:24,047][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:29,048][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:34,049][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:39,050][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:44,051][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:49,052][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:50,457][ ERROR][PID:2418953] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-07 13:50:54,054][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:50:59,055][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:51:00,458][WARNING][PID:2418953] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-07 13:51:04,056][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:51:09,057][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:51:10,467][ INFO][PID:2418953] Downloading the builder-live.log file, attempt 4 [2025-07-07 13:51:10,476][ INFO][PID:2418953] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@18.209.100.254 copr-rpmbuild-log [2025-07-07 13:51:14,058][ INFO][PID:2418953] Checking for cancel request [2025-07-07 13:52:10,568][ ERROR][PID:2418953] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-07 13:52:14,135][ ERROR][PID:2418953] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 18.209.100.254 port 22: Connection timed out [2025-07-07 13:52:20,570][ ERROR][PID:2418953] Giving up for unstable SSH, failures: 1751896100.3145394, 1751896180.3875868, 1751896260.458577, 1751896340.5697298 [2025-07-07 13:53:24,204][ ERROR][PID:2418953] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 18.209.100.254 port 22: Connection timed out [2025-07-07 13:54:34,264][ ERROR][PID:2418953] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 18.209.100.254 port 22: Connection timed out [2025-07-07 13:55:44,340][ ERROR][PID:2418953] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 18.209.100.254 port 22: Connection timed out [2025-07-07 13:55:54,343][ ERROR][PID:2418953] Can't ssh to cancel build. [2025-07-07 13:55:54,344][ ERROR][PID:2418953] Build was canceled [2025-07-07 13:55:54,345][ INFO][PID:2418953] Releasing VM back to pool [2025-07-07 13:55:54,362][WARNING][PID:2418953] Switching not-finished job state to 'failed' [2025-07-07 13:55:54,364][ INFO][PID:2418953] Worker failed build, took 741.7883725166321 [2025-07-07 13:55:54,366][ INFO][PID:2418953] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "packit", "project_name": "osbuild-osbuild-composer-4764", "project_dirname": "osbuild-osbuild-composer-4764", "submitter": "packit", "ended_on": 1751896554.3624837, "started_on": 1751895812.5741112, "submitted_on": null, "status": 0, "chroot": "fedora-rawhide-aarch64", "arch": "aarch64", "buildroot_pkgs": [], "task_id": "9245360-fedora-rawhide-aarch64", "build_id": 9245360, "package_name": "osbuild-composer", "package_version": "101-1.20250707134236630338.pr4764.2.gc7e6058f2", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/packit/osbuild-osbuild-composer-4764/osbuild-composer", "git_hash": "f7f9fc3b958b3edcb1de704b96af9c4acfa8112e", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "packit/osbuild-osbuild-composer-4764--packit", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/packit/osbuild-osbuild-composer-4764/fedora-rawhide-aarch64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_aarch64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/packit/osbuild-osbuild-composer-4764", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/packit/osbuild-osbuild-composer-4764", "result_dir": "09245360-osbuild-composer", "built_packages": "", "id": 9245360, "mockchain_macros": { "copr_username": "packit", "copr_projectname": "osbuild-osbuild-composer-4764", "vendor": "Fedora Project COPR (packit/osbuild-osbuild-composer-4764)" } } ] } [2025-07-07 13:55:54,422][ INFO][PID:2418953] Sending fedora-messaging bus message in build.end [2025-07-07 13:55:54,454][ INFO][PID:2418953] Compressing /var/lib/copr/public_html/results/packit/osbuild-osbuild-composer-4764/fedora-rawhide-aarch64/09245360-osbuild-composer/builder-live.log by gzip [2025-07-07 13:55:54,457][ INFO][PID:2418953] Running command 'gzip /var/lib/copr/public_html/results/packit/osbuild-osbuild-composer-4764/fedora-rawhide-aarch64/09245360-osbuild-composer/builder-live.log' as PID 2513088 [2025-07-07 13:55:54,459][ INFO][PID:2418953] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/packit/osbuild-osbuild-composer-4764/fedora-rawhide-aarch64/09245360-osbuild-composer/builder-live.log) [2025-07-07 13:55:54,460][ INFO][PID:2418953] Compressing /var/lib/copr/public_html/results/packit/osbuild-osbuild-composer-4764/fedora-rawhide-aarch64/09245360-osbuild-composer/backend.log by gzip [2025-07-07 13:55:54,464][ INFO][PID:2418953] Running command 'gzip /var/lib/copr/public_html/results/packit/osbuild-osbuild-composer-4764/fedora-rawhide-aarch64/09245360-osbuild-composer/backend.log' as PID 2513089