[2025-07-23 01:22:50,076][ INFO][PID:1637078] Marking build as starting [2025-07-23 01:22:50,278][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:22:50,279][ INFO][PID:1637078] VM allocation process starts [2025-07-23 01:22:50,297][ INFO][PID:1637078] Trying to allocate VM: ResallocHost, ticket_id=4045670, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-23 01:22:53,320][ INFO][PID:1637078] Allocated host ResallocHost, ticket_id=4045670, hostname=140.211.168.158, name=copr_osuosl_p09_prod_05743848_20250723_000824, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-23 01:22:53,321][ INFO][PID:1637078] Allocating ssh connection to builder [2025-07-23 01:22:53,322][ INFO][PID:1637078] Checking that builder machine is OK [2025-07-23 01:22:55,017][ INFO][PID:1637078] Running remote command: copr-builder-ready fedora-eln-ppc64le [2025-07-23 01:22:56,095][ INFO][PID:1637078] Red Hat subscription not needed for fedora-eln-ppc64le Builder is ready to be used [2025-07-23 01:22:56,096][ INFO][PID:1637078] Filling build.info file with builder info [2025-07-23 01:22:56,097][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:22:56,099][ INFO][PID:1637078] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "rpmsoftwaremanagement", "project_name": "dnf-nightly", "project_dirname": "dnf-nightly", "submitter": "rpmsoftwaremanagement", "ended_on": null, "started_on": 1753233776.0991209, "submitted_on": null, "status": 3, "chroot": "fedora-eln-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "9306937-fedora-eln-ppc64le", "build_id": 9306937, "package_name": "drpm", "package_version": "0.5.2-20250723012102.0.g48c0fde", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rpmsoftwaremanagement/dnf-nightly/drpm", "git_hash": "d9cb6da0e90e00de4684faff8eba5764300b2635", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rpmsoftwaremanagement/dnf-nightly--rpmsoftwaremanagement", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://kojipkgs.fedoraproject.org/repos/eln-build/latest/$basearch", "id": "https_kojipkgs_fedoraproject_org_repos_eln_build_latest_basearch", "name": "Additional repo https_kojipkgs_fedoraproject_org_repos_eln_build_latest_basearch" } ], "background": false, "bootstrap": "off", "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_ppc64le", "arch_power9" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf-nightly", "result_dir": "09306937-drpm", "built_packages": "", "id": 9306937, "mockchain_macros": { "copr_username": "rpmsoftwaremanagement", "copr_projectname": "dnf-nightly", "vendor": "Fedora Project COPR (rpmsoftwaremanagement/dnf-nightly)" } } ] } [2025-07-23 01:22:56,170][ INFO][PID:1637078] Sending fedora-messaging bus message in build.start [2025-07-23 01:22:56,988][ INFO][PID:1637078] Sending fedora-messaging bus message in chroot.start [2025-07-23 01:22:57,016][ INFO][PID:1637078] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9306937-fedora-eln-ppc64le --chroot fedora-eln-ppc64le --detached [2025-07-23 01:22:58,386][ INFO][PID:1637078] Downloading the builder-live.log file, attempt 1 [2025-07-23 01:22:58,388][ INFO][PID:1637078] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2025-07-23 01:23:03,396][ INFO][PID:1637078] Periodic builder liveness probe: alive [2025-07-23 01:23:03,398][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:08,400][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:13,401][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:18,404][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:23,405][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:28,407][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:33,409][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:38,416][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:43,421][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:48,426][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:53,428][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:23:58,431][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:03,433][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:08,436][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:13,439][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:18,441][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:23,443][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:28,444][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:33,446][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:38,449][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:43,453][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:48,455][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:53,456][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:24:58,458][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:03,460][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:08,462][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:13,465][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:18,466][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:23,468][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:28,475][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:33,477][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:38,481][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:43,483][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:48,486][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:53,488][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:25:58,490][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:03,493][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:08,494][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:13,498][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:18,499][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:22,139][ ERROR][PID:1637078] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-23 01:26:23,503][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:28,505][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:32,141][WARNING][PID:1637078] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-23 01:26:33,507][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:38,513][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:42,142][ INFO][PID:1637078] Downloading the builder-live.log file, attempt 2 [2025-07-23 01:26:42,145][ INFO][PID:1637078] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2025-07-23 01:26:43,515][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:48,516][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:53,517][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:26:58,519][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:03,528][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:08,530][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:13,532][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:18,533][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:23,535][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:28,538][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:33,541][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:38,543][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:42,288][ ERROR][PID:1637078] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-23 01:27:43,548][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:48,549][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:52,297][WARNING][PID:1637078] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-23 01:27:53,551][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:27:58,553][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:02,302][ INFO][PID:1637078] Downloading the builder-live.log file, attempt 3 [2025-07-23 01:28:02,305][ INFO][PID:1637078] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2025-07-23 01:28:03,555][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:08,557][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:13,563][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:18,564][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:23,570][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:28,572][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:33,574][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:38,577][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:43,579][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:48,580][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:53,583][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:28:58,584][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:02,447][ ERROR][PID:1637078] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-23 01:29:03,586][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:08,588][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:12,448][WARNING][PID:1637078] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-23 01:29:13,590][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:18,593][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:22,453][ INFO][PID:1637078] Downloading the builder-live.log file, attempt 4 [2025-07-23 01:29:22,455][ INFO][PID:1637078] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2025-07-23 01:29:23,596][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:28,597][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:33,599][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:38,601][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:43,602][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:48,607][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:53,608][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:29:58,610][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:03,611][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:08,613][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:13,614][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:18,615][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:22,600][ ERROR][PID:1637078] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-23 01:30:23,620][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:28,621][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:32,603][ ERROR][PID:1637078] Giving up for unstable SSH, failures: 1753233992.1415768, 1753234072.2971134, 1753234152.448893, 1753234232.6029735 [2025-07-23 01:30:32,604][ ERROR][PID:1637078] Re-try request for task on 'ResallocHost, ticket_id=4045670, hostname=140.211.168.158, name=copr_osuosl_p09_prod_05743848_20250723_000824, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-07-23 01:30:32,604][ INFO][PID:1637078] Releasing VM back to pool [2025-07-23 01:30:32,618][ INFO][PID:1637078] Retry #1 (on other host) [2025-07-23 01:30:32,620][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:32,621][ INFO][PID:1637078] VM allocation process starts [2025-07-23 01:30:32,634][ INFO][PID:1637078] Trying to allocate VM: ResallocHost, ticket_id=4045875, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-23 01:30:35,649][ INFO][PID:1637078] Allocated host ResallocHost, ticket_id=4045875, hostname=140.211.168.248, name=copr_osuosl_p09_prod_05745654_20250723_012252, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-23 01:30:35,649][ INFO][PID:1637078] Allocating ssh connection to builder [2025-07-23 01:30:35,650][ INFO][PID:1637078] Checking that builder machine is OK [2025-07-23 01:30:36,919][ INFO][PID:1637078] Running remote command: copr-builder-ready fedora-eln-ppc64le [2025-07-23 01:30:37,915][ INFO][PID:1637078] Red Hat subscription not needed for fedora-eln-ppc64le Builder is ready to be used [2025-07-23 01:30:37,915][ INFO][PID:1637078] Filling build.info file with builder info [2025-07-23 01:30:37,918][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:37,919][ INFO][PID:1637078] Not re-notifying FE and msg buses for the new host. [2025-07-23 01:30:37,920][ INFO][PID:1637078] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9306937-fedora-eln-ppc64le --chroot fedora-eln-ppc64le --detached [2025-07-23 01:30:39,190][ INFO][PID:1637078] Downloading the builder-live.log file, attempt 1 [2025-07-23 01:30:39,192][ INFO][PID:1637078] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.248 copr-rpmbuild-log [2025-07-23 01:30:44,193][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:49,197][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:54,199][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:30:59,200][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:31:04,202][ INFO][PID:1637078] Checking for cancel request [2025-07-23 01:31:05,701][ INFO][PID:1637078] Downloading results from builder [2025-07-23 01:31:05,702][ INFO][PID:1637078] rsyncing of mockbuilder@140.211.168.248:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09306937-drpm started [2025-07-23 01:31:05,706][ INFO][PID:1637078] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@140.211.168.248:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09306937-drpm/ &> /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09306937-drpm/build-09306937.rsync.log [2025-07-23 01:31:07,249][ INFO][PID:1637078] rsyncing finished. [2025-07-23 01:31:07,249][ INFO][PID:1637078] Releasing VM back to pool [2025-07-23 01:31:07,267][ INFO][PID:1637078] Searching for 'success' file in resultdir [2025-07-23 01:31:07,268][ ERROR][PID:1637078] Build failed: Backend process error: No success file => build failure [2025-07-23 01:31:07,269][ INFO][PID:1637078] Finished build: id=9306937 failed=True timeout=18000 destdir=/var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly chroot=fedora-eln-ppc64le [2025-07-23 01:31:07,272][ ERROR][PID:1637078] Backend process error: No success file => build failure [2025-07-23 01:31:07,273][ INFO][PID:1637078] Worker failed build, took 29.353590488433838 [2025-07-23 01:31:07,276][ INFO][PID:1637078] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "rpmsoftwaremanagement", "project_name": "dnf-nightly", "project_dirname": "dnf-nightly", "submitter": "rpmsoftwaremanagement", "ended_on": 1753234267.2733862, "started_on": 1753234237.9197958, "submitted_on": null, "status": 0, "chroot": "fedora-eln-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "9306937-fedora-eln-ppc64le", "build_id": 9306937, "package_name": "drpm", "package_version": "0.5.2-20250723012102.0.g48c0fde", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rpmsoftwaremanagement/dnf-nightly/drpm", "git_hash": "d9cb6da0e90e00de4684faff8eba5764300b2635", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rpmsoftwaremanagement/dnf-nightly--rpmsoftwaremanagement", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://kojipkgs.fedoraproject.org/repos/eln-build/latest/$basearch", "id": "https_kojipkgs_fedoraproject_org_repos_eln_build_latest_basearch", "name": "Additional repo https_kojipkgs_fedoraproject_org_repos_eln_build_latest_basearch" } ], "background": false, "bootstrap": "off", "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_ppc64le", "arch_power9" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf-nightly", "result_dir": "09306937-drpm", "built_packages": "", "id": 9306937, "mockchain_macros": { "copr_username": "rpmsoftwaremanagement", "copr_projectname": "dnf-nightly", "vendor": "Fedora Project COPR (rpmsoftwaremanagement/dnf-nightly)" } } ] } [2025-07-23 01:31:07,374][ INFO][PID:1637078] Sending fedora-messaging bus message in build.end [2025-07-23 01:31:07,402][ INFO][PID:1637078] Compressing /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09306937-drpm/builder-live.log by gzip [2025-07-23 01:31:07,404][ INFO][PID:1637078] Running command 'gzip /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09306937-drpm/builder-live.log' as PID 1711878 [2025-07-23 01:31:07,410][ INFO][PID:1637078] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09306937-drpm/builder-live.log) [2025-07-23 01:31:07,411][ INFO][PID:1637078] Compressing /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09306937-drpm/backend.log by gzip [2025-07-23 01:31:07,413][ INFO][PID:1637078] Running command 'gzip /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09306937-drpm/backend.log' as PID 1711880