[2025-07-14 18:25:07,274][ INFO][PID:1077559] Marking build as starting [2025-07-14 18:25:07,341][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:07,342][ INFO][PID:1077559] VM allocation process starts [2025-07-14 18:25:07,353][ INFO][PID:1077559] Trying to allocate VM: ResallocHost, ticket_id=3924807, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-14 18:25:10,377][ INFO][PID:1077559] Allocated host ResallocHost, ticket_id=3924807, hostname=34.238.120.114, name=aws_x86_64_spot_prod_05386213_20250714_182112, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-14 18:25:10,378][ INFO][PID:1077559] Allocating ssh connection to builder [2025-07-14 18:25:10,379][ INFO][PID:1077559] Checking that builder machine is OK [2025-07-14 18:25:10,892][ INFO][PID:1077559] Running remote command: copr-builder-ready fedora-42-x86_64 [2025-07-14 18:25:11,100][ INFO][PID:1077559] Red Hat subscription not needed for fedora-42-x86_64 Builder is ready to be used [2025-07-14 18:25:11,101][ INFO][PID:1077559] Filling build.info file with builder info [2025-07-14 18:25:11,102][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:11,103][ INFO][PID:1077559] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "packit", "project_name": "containers-podman-26330", "project_dirname": "containers-podman-26330", "submitter": "packit", "ended_on": null, "started_on": 1752517511.1034162, "submitted_on": null, "status": 3, "chroot": "fedora-42-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "9272957-fedora-42-x86_64", "build_id": 9272957, "package_name": "podman", "package_version": "5:5.6.0~dev-1.20250714182429043238.pr26330.1249.3b2a72fcd4", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/packit/containers-podman-26330/podman", "git_hash": "a8f6a70cd2ef62a6211d8869017307c420e00a56", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "packit/containers-podman-26330--packit", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/packit/containers-podman-26330/fedora-42-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/packit/containers-podman-26330", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/packit/containers-podman-26330", "result_dir": "09272957-podman", "built_packages": "", "id": 9272957, "mockchain_macros": { "copr_username": "packit", "copr_projectname": "containers-podman-26330", "vendor": "Fedora Project COPR (packit/containers-podman-26330)" } } ] } [2025-07-14 18:25:11,161][ INFO][PID:1077559] Sending fedora-messaging bus message in build.start [2025-07-14 18:25:12,180][ INFO][PID:1077559] Sending fedora-messaging bus message in chroot.start [2025-07-14 18:25:12,229][ INFO][PID:1077559] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9272957-fedora-42-x86_64 --chroot fedora-42-x86_64 --detached [2025-07-14 18:25:12,578][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 1 [2025-07-14 18:25:12,580][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.238.120.114 copr-rpmbuild-log [2025-07-14 18:25:17,589][ INFO][PID:1077559] Periodic builder liveness probe: alive [2025-07-14 18:25:17,590][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:22,591][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:27,593][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:32,595][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:37,596][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:42,597][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:47,598][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:52,600][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:25:57,601][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:02,602][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:07,603][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:12,604][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:17,606][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:22,607][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:27,608][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:32,609][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:37,614][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:42,617][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:47,618][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:52,619][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:26:57,620][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:02,621][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:07,622][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:12,624][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:17,625][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:22,626][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:27,628][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:32,629][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:37,630][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:42,631][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:47,632][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:52,633][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:27:57,634][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:02,636][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:07,636][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:12,638][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:17,639][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:22,640][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:27,641][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:32,642][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:37,644][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:42,645][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:47,646][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:52,648][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:57,649][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:28:59,322][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:29:02,650][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:07,656][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:09,323][WARNING][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-14 18:29:12,657][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:17,659][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:19,325][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 2 [2025-07-14 18:29:19,329][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.238.120.114 copr-rpmbuild-log [2025-07-14 18:29:22,661][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:27,667][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:32,671][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:37,672][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:42,673][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:47,674][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:52,676][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:29:57,677][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:02,678][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:07,679][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:12,680][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:17,681][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:19,398][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:30:22,682][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:27,683][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:29,509][WARNING][PID:1077559] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-14 18:30:32,685][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:37,687][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:39,510][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 3 [2025-07-14 18:30:39,513][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.238.120.114 copr-rpmbuild-log [2025-07-14 18:30:42,688][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:47,689][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:52,691][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:30:57,694][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:02,695][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:07,697][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:12,698][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:17,700][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:22,701][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:27,702][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:32,703][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:37,705][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:39,592][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:31:42,713][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:47,720][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:49,611][WARNING][PID:1077559] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-14 18:31:52,722][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:57,724][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:31:59,612][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 4 [2025-07-14 18:31:59,613][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.238.120.114 copr-rpmbuild-log [2025-07-14 18:32:02,725][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:07,727][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:12,728][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:17,729][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:22,731][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:27,733][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:32,739][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:37,744][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:42,752][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:47,753][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:52,754][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:57,755][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:32:59,687][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:33:02,758][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:07,761][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:09,694][ ERROR][PID:1077559] Giving up for unstable SSH, failures: 1752517749.3235703, 1752517829.5097945, 1752517909.611493, 1752517989.6944778 [2025-07-14 18:33:09,696][ ERROR][PID:1077559] Re-try request for task on 'ResallocHost, ticket_id=3924807, hostname=34.238.120.114, name=aws_x86_64_spot_prod_05386213_20250714_182112, requested_tags=['copr_builder', 'arch_x86_64']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-07-14 18:33:09,698][ INFO][PID:1077559] Releasing VM back to pool [2025-07-14 18:33:09,736][ INFO][PID:1077559] Retry #1 (on other host) [2025-07-14 18:33:09,737][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:09,739][ INFO][PID:1077559] VM allocation process starts [2025-07-14 18:33:09,779][ INFO][PID:1077559] Trying to allocate VM: ResallocHost, ticket_id=3924942, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-14 18:33:12,831][ INFO][PID:1077559] Allocated host ResallocHost, ticket_id=3924942, hostname=54.198.198.30, name=aws_x86_64_spot_prod_05386546_20250714_183058, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-14 18:33:12,832][ INFO][PID:1077559] Allocating ssh connection to builder [2025-07-14 18:33:12,834][ INFO][PID:1077559] Checking that builder machine is OK [2025-07-14 18:33:13,323][ INFO][PID:1077559] Running remote command: copr-builder-ready fedora-42-x86_64 [2025-07-14 18:33:13,584][ INFO][PID:1077559] Red Hat subscription not needed for fedora-42-x86_64 Builder is ready to be used [2025-07-14 18:33:13,586][ INFO][PID:1077559] Filling build.info file with builder info [2025-07-14 18:33:13,663][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:13,666][ INFO][PID:1077559] Not re-notifying FE and msg buses for the new host. [2025-07-14 18:33:13,668][ INFO][PID:1077559] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9272957-fedora-42-x86_64 --chroot fedora-42-x86_64 --detached [2025-07-14 18:33:14,027][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 1 [2025-07-14 18:33:14,034][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.198.198.30 copr-rpmbuild-log [2025-07-14 18:33:19,028][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:24,030][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:29,034][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:34,036][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:39,040][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:44,043][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:49,049][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:54,052][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:33:59,053][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:04,056][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:09,057][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:09,459][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:34:14,062][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:19,063][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:19,505][WARNING][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-14 18:34:24,068][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:29,069][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:29,507][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 2 [2025-07-14 18:34:29,510][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.198.198.30 copr-rpmbuild-log [2025-07-14 18:34:34,070][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:39,072][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:44,078][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:49,080][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:54,081][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:34:59,083][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:04,087][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:09,088][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:14,089][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:19,100][ INFO][PID:1077559] Periodic builder liveness probe: alive [2025-07-14 18:35:19,101][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:24,103][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:29,105][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:29,578][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:35:34,106][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:39,111][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:39,720][WARNING][PID:1077559] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-14 18:35:44,115][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:49,117][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:49,721][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 3 [2025-07-14 18:35:49,723][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.198.198.30 copr-rpmbuild-log [2025-07-14 18:35:54,118][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:35:59,128][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:04,129][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:09,130][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:14,132][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:19,137][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:24,138][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:29,139][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:34,140][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:39,142][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:44,144][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:49,145][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:49,792][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:36:54,147][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:59,148][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:36:59,794][WARNING][PID:1077559] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-14 18:37:04,150][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:09,151][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:09,794][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 4 [2025-07-14 18:37:09,796][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.198.198.30 copr-rpmbuild-log [2025-07-14 18:37:14,153][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:19,168][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:24,170][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:29,171][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:34,172][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:39,173][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:44,174][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:49,175][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:54,176][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:37:59,177][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:04,180][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:09,181][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:09,864][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:38:14,182][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:19,183][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:19,865][ ERROR][PID:1077559] Giving up for unstable SSH, failures: 1752518059.505667, 1752518139.7200322, 1752518219.7941206, 1752518299.8656614 [2025-07-14 18:38:19,867][ ERROR][PID:1077559] Re-try request for task on 'ResallocHost, ticket_id=3924942, hostname=54.198.198.30, name=aws_x86_64_spot_prod_05386546_20250714_183058, requested_tags=['copr_builder', 'arch_x86_64']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-07-14 18:38:19,870][ INFO][PID:1077559] Releasing VM back to pool [2025-07-14 18:38:19,885][ INFO][PID:1077559] Retry #2 (on other host) [2025-07-14 18:38:19,886][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:19,888][ INFO][PID:1077559] VM allocation process starts [2025-07-14 18:38:19,908][ INFO][PID:1077559] Trying to allocate VM: ResallocHost, ticket_id=3925020, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-14 18:38:22,977][ INFO][PID:1077559] Allocated host ResallocHost, ticket_id=3925020, hostname=98.82.130.112, name=aws_x86_64_spot_prod_05386986_20250714_183614, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-14 18:38:22,983][ INFO][PID:1077559] Allocating ssh connection to builder [2025-07-14 18:38:22,991][ INFO][PID:1077559] Checking that builder machine is OK [2025-07-14 18:38:23,554][ INFO][PID:1077559] Running remote command: copr-builder-ready fedora-42-x86_64 [2025-07-14 18:38:23,824][ INFO][PID:1077559] Red Hat subscription not needed for fedora-42-x86_64 Builder is ready to be used [2025-07-14 18:38:23,825][ INFO][PID:1077559] Filling build.info file with builder info [2025-07-14 18:38:23,842][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:23,854][ INFO][PID:1077559] Not re-notifying FE and msg buses for the new host. [2025-07-14 18:38:23,857][ INFO][PID:1077559] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9272957-fedora-42-x86_64 --chroot fedora-42-x86_64 --detached [2025-07-14 18:38:24,309][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 1 [2025-07-14 18:38:24,315][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.82.130.112 copr-rpmbuild-log [2025-07-14 18:38:29,315][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:34,318][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:39,320][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:44,321][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:49,323][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:54,324][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:38:59,325][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:04,328][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:09,329][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:14,334][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:19,335][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:24,339][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:29,341][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:29,464][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:39:34,346][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:39,355][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:39,509][WARNING][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-14 18:39:44,358][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:49,363][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:49,510][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 2 [2025-07-14 18:39:49,512][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.82.130.112 copr-rpmbuild-log [2025-07-14 18:39:54,364][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:39:59,368][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:40:04,370][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:40:09,374][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:40:14,379][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:40:19,384][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:40:24,390][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:40:29,392][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:40:34,396][ INFO][PID:1077559] Checking for cancel request [2025-07-14 18:40:49,587][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:40:59,621][WARNING][PID:1077559] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-14 18:41:09,622][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 3 [2025-07-14 18:41:09,623][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.82.130.112 copr-rpmbuild-log [2025-07-14 18:41:34,489][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 98.82.130.112 port 22: Connection timed out [2025-07-14 18:42:09,691][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:42:19,695][WARNING][PID:1077559] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-14 18:42:29,695][ INFO][PID:1077559] Downloading the builder-live.log file, attempt 4 [2025-07-14 18:42:29,700][ INFO][PID:1077559] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.82.130.112 copr-rpmbuild-log [2025-07-14 18:42:44,577][ ERROR][PID:1077559] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 98.82.130.112 port 22: Connection timed out [2025-07-14 18:43:29,770][ ERROR][PID:1077559] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-14 18:43:39,795][ ERROR][PID:1077559] Giving up for unstable SSH, failures: 1752518379.5090187, 1752518459.6213548, 1752518539.6951149, 1752518619.7952487 [2025-07-14 18:43:54,669][ ERROR][PID:1077559] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 98.82.130.112 port 22: Connection timed out [2025-07-14 18:45:04,741][ ERROR][PID:1077559] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 98.82.130.112 port 22: Connection timed out [2025-07-14 18:45:14,742][ ERROR][PID:1077559] Can't ssh to cancel build. [2025-07-14 18:45:14,744][ ERROR][PID:1077559] Build was canceled [2025-07-14 18:45:14,744][ INFO][PID:1077559] Releasing VM back to pool [2025-07-14 18:45:14,754][WARNING][PID:1077559] Switching not-finished job state to 'failed' [2025-07-14 18:45:14,757][ INFO][PID:1077559] Worker failed build, took 410.8997709751129 [2025-07-14 18:45:14,758][ INFO][PID:1077559] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "packit", "project_name": "containers-podman-26330", "project_dirname": "containers-podman-26330", "submitter": "packit", "ended_on": 1752518714.7546442, "started_on": 1752518303.8548732, "submitted_on": null, "status": 0, "chroot": "fedora-42-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "9272957-fedora-42-x86_64", "build_id": 9272957, "package_name": "podman", "package_version": "5:5.6.0~dev-1.20250714182429043238.pr26330.1249.3b2a72fcd4", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/packit/containers-podman-26330/podman", "git_hash": "a8f6a70cd2ef62a6211d8869017307c420e00a56", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "packit/containers-podman-26330--packit", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/packit/containers-podman-26330/fedora-42-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/packit/containers-podman-26330", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/packit/containers-podman-26330", "result_dir": "09272957-podman", "built_packages": "", "id": 9272957, "mockchain_macros": { "copr_username": "packit", "copr_projectname": "containers-podman-26330", "vendor": "Fedora Project COPR (packit/containers-podman-26330)" } } ] } [2025-07-14 18:45:14,797][ INFO][PID:1077559] Sending fedora-messaging bus message in build.end [2025-07-14 18:45:14,847][ INFO][PID:1077559] Compressing /var/lib/copr/public_html/results/packit/containers-podman-26330/fedora-42-x86_64/09272957-podman/builder-live.log by gzip [2025-07-14 18:45:14,852][ INFO][PID:1077559] Running command 'gzip /var/lib/copr/public_html/results/packit/containers-podman-26330/fedora-42-x86_64/09272957-podman/builder-live.log' as PID 1222546 [2025-07-14 18:45:14,854][ INFO][PID:1077559] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/packit/containers-podman-26330/fedora-42-x86_64/09272957-podman/builder-live.log) [2025-07-14 18:45:14,855][ INFO][PID:1077559] Compressing /var/lib/copr/public_html/results/packit/containers-podman-26330/fedora-42-x86_64/09272957-podman/backend.log by gzip [2025-07-14 18:45:14,857][ INFO][PID:1077559] Running command 'gzip /var/lib/copr/public_html/results/packit/containers-podman-26330/fedora-42-x86_64/09272957-podman/backend.log' as PID 1222550