[2025-07-24 04:56:42,178][ INFO][PID:1396236] Marking build as starting [2025-07-24 04:56:42,251][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:56:42,253][ INFO][PID:1396236] VM allocation process starts [2025-07-24 04:56:42,287][ INFO][PID:1396236] Trying to allocate VM: ResallocHost, ticket_id=4086385, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-24 04:56:47,291][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:56:48,338][ INFO][PID:1396236] Allocated host ResallocHost, ticket_id=4086385, hostname=140.211.168.80, name=copr_osuosl_p09_prod_05777634_20250724_044339, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-24 04:56:48,338][ INFO][PID:1396236] Allocating ssh connection to builder [2025-07-24 04:56:48,339][ INFO][PID:1396236] Checking that builder machine is OK [2025-07-24 04:56:49,738][ INFO][PID:1396236] Running remote command: copr-builder-ready fedora-eln-ppc64le [2025-07-24 04:56:50,743][ INFO][PID:1396236] Red Hat subscription not needed for fedora-eln-ppc64le Builder is ready to be used [2025-07-24 04:56:50,744][ INFO][PID:1396236] Filling build.info file with builder info [2025-07-24 04:56:50,745][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:56:50,746][ INFO][PID:1396236] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "rpmsoftwaremanagement", "project_name": "dnf-nightly", "project_dirname": "dnf-nightly", "submitter": "rpmsoftwaremanagement", "ended_on": null, "started_on": 1753333010.7458253, "submitted_on": null, "status": 3, "chroot": "fedora-eln-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "9316868-fedora-eln-ppc64le", "build_id": 9316868, "package_name": "microdnf", "package_version": "3.10.1-20250724012027.1.g26c9397", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rpmsoftwaremanagement/dnf-nightly/microdnf", "git_hash": "88f88d7940766cd57f4a5518e6fd19d3aa10f357", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rpmsoftwaremanagement/dnf-nightly--rpmsoftwaremanagement", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://kojipkgs.fedoraproject.org/repos/eln-build/latest/$basearch", "id": "https_kojipkgs_fedoraproject_org_repos_eln_build_latest_basearch", "name": "Additional repo https_kojipkgs_fedoraproject_org_repos_eln_build_latest_basearch" } ], "background": false, "bootstrap": "off", "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_ppc64le", "arch_power9" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf-nightly", "result_dir": "09316868-microdnf", "built_packages": "", "id": 9316868, "mockchain_macros": { "copr_username": "rpmsoftwaremanagement", "copr_projectname": "dnf-nightly", "vendor": "Fedora Project COPR (rpmsoftwaremanagement/dnf-nightly)" } } ] } [2025-07-24 04:56:50,787][ INFO][PID:1396236] Sending fedora-messaging bus message in build.start [2025-07-24 04:56:51,497][ INFO][PID:1396236] Sending fedora-messaging bus message in chroot.start [2025-07-24 04:56:51,522][ INFO][PID:1396236] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9316868-fedora-eln-ppc64le --chroot fedora-eln-ppc64le --detached [2025-07-24 04:56:52,775][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 1 [2025-07-24 04:56:52,776][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.80 copr-rpmbuild-log [2025-07-24 04:56:57,780][ INFO][PID:1396236] Periodic builder liveness probe: alive [2025-07-24 04:56:57,781][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:02,782][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:07,783][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:12,784][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:17,785][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:22,786][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:27,787][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:32,789][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:37,792][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:42,794][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:47,795][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:52,796][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:57:57,797][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:02,798][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:07,799][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:12,800][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:17,801][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:22,802][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:27,803][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:32,805][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:37,806][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:42,807][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:47,808][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:52,809][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:58:57,810][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:02,812][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:07,813][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:12,815][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:17,818][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:22,832][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:27,833][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:32,834][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:37,835][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:42,836][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:47,838][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:52,839][ INFO][PID:1396236] Checking for cancel request [2025-07-24 04:59:57,841][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:02,843][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:07,846][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:12,848][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:17,849][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:22,850][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:27,853][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:32,857][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:37,859][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:42,861][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:47,863][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:52,866][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:00:57,868][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:02,869][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:07,881][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:12,886][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:17,887][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:22,890][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:27,893][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:32,901][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:37,903][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:42,905][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:47,906][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:52,907][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:01:57,910][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:02,911][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:07,912][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:12,915][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:17,917][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:22,918][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:27,919][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:32,920][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:37,921][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:42,922][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:47,924][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:52,925][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:02:57,926][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:02,927][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:07,930][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:12,931][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:17,932][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:22,933][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:27,934][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:32,935][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:37,936][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:42,940][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:47,941][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:52,943][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:03:57,944][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:02,945][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:07,946][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:12,947][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:17,948][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:22,950][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:27,951][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:32,952][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:37,953][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:42,955][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:47,963][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:52,964][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:04:57,965][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:02,968][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:07,969][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:12,970][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:17,973][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:22,975][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:27,979][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:32,980][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:37,982][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:42,983][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:47,984][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:52,985][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:05:57,986][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:02,987][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:07,988][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:12,989][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:17,991][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:22,992][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:27,993][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:32,997][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:38,000][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:43,001][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:48,003][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:53,004][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:06:58,010][ INFO][PID:1396236] Periodic builder liveness probe: alive [2025-07-24 05:06:58,011][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:03,011][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:08,013][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:13,015][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:18,017][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:23,019][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:28,022][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:33,024][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:38,026][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:43,027][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:48,031][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:53,035][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:07:58,038][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:03,039][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:08,042][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:13,050][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:18,052][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:23,053][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:28,064][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:33,067][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:38,074][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:43,082][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:48,084][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:53,087][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:08:58,090][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:03,093][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:08,099][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:13,109][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:18,112][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:23,114][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:28,116][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:33,120][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:38,122][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:43,124][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:48,127][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:53,132][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:09:58,134][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:03,136][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:08,141][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:13,145][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:18,152][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:23,157][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:28,160][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:33,163][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:38,169][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:43,196][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:48,199][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:53,211][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:10:58,213][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:03,228][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:08,233][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:13,235][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:18,241][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:23,250][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:28,254][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:33,258][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:38,264][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:43,271][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:48,275][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:53,280][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:11:58,282][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:03,284][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:08,288][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:13,292][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:18,297][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:23,299][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:28,302][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:33,303][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:38,304][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:43,306][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:48,307][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:53,308][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:12:58,309][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:03,311][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:08,312][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:13,313][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:18,314][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:23,315][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:28,316][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:33,317][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:38,318][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:43,320][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:48,321][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:53,323][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:13:58,324][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:03,325][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:08,326][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:13,327][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:18,329][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:23,330][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:28,331][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:33,332][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:38,333][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:43,334][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:48,336][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:53,337][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:14:58,340][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:03,341][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:08,343][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:13,344][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:18,345][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:23,346][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:28,347][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:33,348][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:38,349][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:43,351][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:48,352][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:53,354][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:15:58,356][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:03,357][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:08,358][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:13,359][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:18,360][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:23,361][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:28,363][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:33,364][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:38,366][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:43,367][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:48,368][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:53,369][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:16:58,379][ INFO][PID:1396236] Periodic builder liveness probe: alive [2025-07-24 05:16:58,380][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:03,381][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:08,382][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:13,383][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:18,385][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:23,386][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:28,387][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:33,388][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:38,389][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:43,391][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:48,392][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:53,393][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:17:58,394][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:03,395][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:08,397][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:13,398][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:18,399][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:23,400][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:28,402][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:33,403][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:38,404][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:43,405][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:48,406][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:53,408][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:18:58,410][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:03,412][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:08,413][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:13,415][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:18,416][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:23,418][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:28,419][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:33,420][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:38,421][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:43,423][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:48,425][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:53,447][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:19:58,448][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:03,449][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:08,451][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:13,452][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:18,453][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:23,456][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:28,458][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:33,461][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:38,463][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:43,468][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:48,469][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:53,470][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:20:58,471][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:03,473][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:08,474][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:13,481][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:18,482][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:23,484][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:28,485][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:33,487][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:38,489][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:43,489][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:48,490][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:53,492][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:21:58,493][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:03,494][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:08,495][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:13,498][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:18,499][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:23,500][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:28,502][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:33,503][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:38,504][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:43,505][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:48,506][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:53,507][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:22:58,508][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:03,509][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:08,510][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:13,511][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:18,513][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:23,514][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:28,515][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:33,516][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:38,518][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:43,519][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:48,520][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:53,521][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:23:58,522][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:03,523][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:08,525][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:13,526][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:18,527][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:23,529][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:28,530][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:33,531][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:38,532][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:43,533][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:48,534][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:53,536][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:24:58,537][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:03,539][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:08,540][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:13,541][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:18,542][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:23,543][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:28,545][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:33,546][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:38,547][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:43,548][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:48,549][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:53,550][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:25:58,552][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:03,559][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:08,560][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:13,562][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:18,563][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:23,564][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:28,566][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:33,570][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:38,572][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:43,574][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:48,576][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:53,581][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:26:58,633][ INFO][PID:1396236] Periodic builder liveness probe: dead [2025-07-24 05:27:58,711][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.80 port 22: Connection timed out [2025-07-24 05:28:06,145][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 05:28:16,146][WARNING][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-24 05:28:26,147][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 2 [2025-07-24 05:28:26,148][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.80 copr-rpmbuild-log [2025-07-24 05:29:08,785][ ERROR][PID:1396236] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.80 port 22: Connection timed out [2025-07-24 05:29:26,217][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 05:29:36,220][WARNING][PID:1396236] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-24 05:29:46,221][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 3 [2025-07-24 05:29:46,222][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.80 copr-rpmbuild-log [2025-07-24 05:30:18,854][ ERROR][PID:1396236] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.80 port 22: Connection timed out [2025-07-24 05:30:46,290][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 05:30:56,293][WARNING][PID:1396236] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-24 05:31:06,294][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 4 [2025-07-24 05:31:06,295][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.80 copr-rpmbuild-log [2025-07-24 05:31:28,923][ ERROR][PID:1396236] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.80 port 22: Connection timed out [2025-07-24 05:31:38,924][ ERROR][PID:1396236] Can't ssh to cancel build. [2025-07-24 05:32:06,363][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 05:32:16,367][ ERROR][PID:1396236] Giving up for unstable SSH, failures: 1753334896.146657, 1753334976.220596, 1753335056.2938569, 1753335136.3671327 [2025-07-24 05:32:16,368][ ERROR][PID:1396236] Re-try request for task on 'ResallocHost, ticket_id=4086385, hostname=140.211.168.80, name=copr_osuosl_p09_prod_05777634_20250724_044339, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-07-24 05:32:16,368][ INFO][PID:1396236] Releasing VM back to pool [2025-07-24 05:32:16,378][ INFO][PID:1396236] Retry #1 (on other host) [2025-07-24 05:32:16,379][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:32:16,379][ INFO][PID:1396236] VM allocation process starts [2025-07-24 05:32:16,390][ INFO][PID:1396236] Trying to allocate VM: ResallocHost, ticket_id=4086803, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-24 05:32:21,391][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:32:26,392][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:32:31,393][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:32:36,394][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:32:41,396][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:32:46,397][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:32:51,398][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:32:56,399][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:01,400][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:06,401][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:11,402][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:16,404][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:21,405][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:26,406][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:31,407][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:36,408][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:41,409][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:46,410][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:51,411][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:33:56,413][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:01,414][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:06,415][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:11,416][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:16,417][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:21,418][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:26,419][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:31,421][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:36,422][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:41,423][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:46,424][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:51,425][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:34:56,425][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:01,426][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:06,428][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:11,428][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:16,430][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:21,431][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:26,432][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:31,433][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:36,434][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:41,435][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:46,436][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:51,437][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:35:56,438][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:01,439][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:06,442][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:11,443][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:16,445][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:21,446][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:26,448][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:31,449][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:36,450][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:41,451][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:46,452][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:51,453][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:36:56,454][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:01,455][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:06,456][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:11,581][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:16,950][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:22,154][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:27,155][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:32,158][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:37,163][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:42,165][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:47,167][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:52,168][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:37:57,169][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:02,170][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:07,171][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:12,172][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:17,173][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:22,174][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:27,175][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:32,176][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:34,489][ INFO][PID:1396236] Allocated host ResallocHost, ticket_id=4086803, hostname=140.211.168.248, name=copr_osuosl_p09_prod_05778178_20250724_053421, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-24 05:38:34,490][ INFO][PID:1396236] Allocating ssh connection to builder [2025-07-24 05:38:34,490][ INFO][PID:1396236] Checking that builder machine is OK [2025-07-24 05:38:35,938][ INFO][PID:1396236] Running remote command: copr-builder-ready fedora-eln-ppc64le [2025-07-24 05:38:36,921][ INFO][PID:1396236] Red Hat subscription not needed for fedora-eln-ppc64le Builder is ready to be used [2025-07-24 05:38:36,922][ INFO][PID:1396236] Filling build.info file with builder info [2025-07-24 05:38:36,953][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:36,955][ INFO][PID:1396236] Not re-notifying FE and msg buses for the new host. [2025-07-24 05:38:36,956][ INFO][PID:1396236] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9316868-fedora-eln-ppc64le --chroot fedora-eln-ppc64le --detached [2025-07-24 05:38:38,310][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 1 [2025-07-24 05:38:38,314][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.248 copr-rpmbuild-log [2025-07-24 05:38:43,371][ INFO][PID:1396236] Periodic builder liveness probe: alive [2025-07-24 05:38:43,373][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:48,375][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:53,378][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:38:58,379][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:03,380][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:08,383][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:13,385][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:18,776][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:24,059][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:29,244][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:34,620][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:39,897][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:45,108][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:50,405][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:39:55,407][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:00,777][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:05,814][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:10,974][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:16,299][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:21,300][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:26,301][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:31,302][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:36,304][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:41,305][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:46,306][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:51,308][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:40:56,309][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:01,312][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:06,315][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:11,316][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:16,318][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:21,322][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:26,323][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:31,325][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:36,327][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:41,329][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:46,332][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:51,379][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:41:55,139][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 05:41:56,380][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:01,382][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:05,140][WARNING][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-24 05:42:06,525][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:11,541][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:15,141][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 2 [2025-07-24 05:42:15,142][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.248 copr-rpmbuild-log [2025-07-24 05:42:16,586][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:21,790][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:26,793][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:31,954][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:36,982][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:42,315][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:47,548][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:52,571][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:42:57,713][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:02,860][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:07,922][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:13,024][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:15,280][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 05:43:18,262][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:23,341][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:25,293][WARNING][PID:1396236] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-24 05:43:28,745][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:33,747][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:35,294][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 3 [2025-07-24 05:43:35,295][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.248 copr-rpmbuild-log [2025-07-24 05:43:38,748][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:43,750][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:48,755][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:53,759][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:43:58,760][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:03,763][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:08,765][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:13,767][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:18,768][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:23,773][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:28,776][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:33,778][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:35,433][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 05:44:38,783][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:43,785][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:45,434][WARNING][PID:1396236] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-24 05:44:48,788][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:53,790][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:44:55,435][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 4 [2025-07-24 05:44:55,436][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.248 copr-rpmbuild-log [2025-07-24 05:44:58,791][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:03,793][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:09,324][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:14,446][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:19,508][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:24,564][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:29,569][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:34,570][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:39,571][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:44,574][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:49,577][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:54,578][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:45:55,571][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 05:45:59,579][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:04,581][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:05,572][ ERROR][PID:1396236] Giving up for unstable SSH, failures: 1753335725.1406522, 1753335805.2933369, 1753335885.43469, 1753335965.5723393 [2025-07-24 05:46:05,573][ ERROR][PID:1396236] Re-try request for task on 'ResallocHost, ticket_id=4086803, hostname=140.211.168.248, name=copr_osuosl_p09_prod_05778178_20250724_053421, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-07-24 05:46:05,573][ INFO][PID:1396236] Releasing VM back to pool [2025-07-24 05:46:05,583][ INFO][PID:1396236] Retry #2 (on other host) [2025-07-24 05:46:05,584][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:05,585][ INFO][PID:1396236] VM allocation process starts [2025-07-24 05:46:05,629][ INFO][PID:1396236] Trying to allocate VM: ResallocHost, ticket_id=4086917, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-24 05:46:08,656][ INFO][PID:1396236] Allocated host ResallocHost, ticket_id=4086917, hostname=140.211.168.240, name=copr_osuosl_p09_prod_05778202_20250724_053751, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9'] [2025-07-24 05:46:08,656][ INFO][PID:1396236] Allocating ssh connection to builder [2025-07-24 05:46:08,657][ INFO][PID:1396236] Checking that builder machine is OK [2025-07-24 05:46:09,874][ INFO][PID:1396236] Running remote command: copr-builder-ready fedora-eln-ppc64le [2025-07-24 05:46:10,938][ INFO][PID:1396236] Red Hat subscription not needed for fedora-eln-ppc64le Builder is ready to be used [2025-07-24 05:46:10,939][ INFO][PID:1396236] Filling build.info file with builder info [2025-07-24 05:46:10,939][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:10,940][ INFO][PID:1396236] Not re-notifying FE and msg buses for the new host. [2025-07-24 05:46:10,941][ INFO][PID:1396236] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9316868-fedora-eln-ppc64le --chroot fedora-eln-ppc64le --detached [2025-07-24 05:46:12,241][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 1 [2025-07-24 05:46:12,242][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.240 copr-rpmbuild-log [2025-07-24 05:46:17,241][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:22,243][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:27,245][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:32,246][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:37,247][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:42,248][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:47,249][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:52,250][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:46:57,251][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:02,254][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:07,260][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:12,261][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:17,262][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:22,264][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:27,270][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:32,283][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:37,284][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:42,285][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:47,286][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:52,288][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:47:57,289][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:02,290][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:07,291][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:12,293][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:17,294][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:22,295][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:27,296][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:32,297][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:37,298][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:42,299][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:47,308][ INFO][PID:1396236] Periodic builder liveness probe: alive [2025-07-24 05:48:47,309][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:52,310][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:48:57,313][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:02,318][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:07,319][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:12,320][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:17,322][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:22,325][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:27,327][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:32,332][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:37,336][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:42,340][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:47,347][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:52,349][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:49:57,350][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:02,351][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:07,352][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:12,354][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:17,357][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:22,359][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:27,360][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:32,370][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:37,375][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:42,376][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:47,378][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:52,379][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:50:57,380][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:02,381][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:07,382][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:12,384][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:17,387][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:22,391][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:27,392][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:32,393][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:37,395][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:42,396][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:47,397][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:52,398][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:51:57,399][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:02,400][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:07,402][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:12,403][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:17,404][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:22,405][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:27,406][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:32,407][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:37,409][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:42,410][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:47,411][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:52,412][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:52:57,413][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:02,414][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:07,415][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:12,416][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:17,417][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:22,418][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:27,419][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:32,420][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:37,422][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:42,423][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:47,424][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:52,425][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:53:57,427][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:02,428][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:07,429][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:12,433][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:17,434][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:22,437][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:27,439][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:32,440][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:37,442][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:42,444][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:47,445][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:52,446][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:54:57,448][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:02,449][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:07,452][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:12,454][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:17,459][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:22,460][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:27,461][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:32,462][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:37,464][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:42,465][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:47,466][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:52,467][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:55:57,468][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:02,469][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:07,470][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:12,472][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:17,473][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:22,474][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:27,475][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:32,477][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:37,478][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:42,479][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:47,480][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:52,481][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:56:57,482][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:02,483][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:07,484][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:12,486][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:17,487][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:22,488][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:27,489][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:32,490][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:37,491][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:42,493][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:47,494][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:52,495][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:57:57,496][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:02,497][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:07,498][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:12,499][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:17,500][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:22,501][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:27,502][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:32,503][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:37,505][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:42,506][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:47,516][ INFO][PID:1396236] Periodic builder liveness probe: alive [2025-07-24 05:58:47,517][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:52,518][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:58:57,519][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:02,520][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:07,521][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:12,522][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:17,523][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:22,524][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:27,525][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:32,526][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:37,528][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:42,529][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:47,530][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:52,531][ INFO][PID:1396236] Checking for cancel request [2025-07-24 05:59:57,532][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:02,533][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:07,534][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:12,536][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:17,537][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:22,538][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:27,539][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:32,540][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:37,541][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:42,542][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:47,543][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:52,544][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:00:57,546][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:02,547][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:07,548][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:12,549][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:17,550][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:22,552][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:27,553][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:32,554][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:37,555][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:42,556][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:47,557][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:52,558][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:01:57,559][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:02,561][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:07,562][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:12,563][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:17,564][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:22,566][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:27,572][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:32,574][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:37,576][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:42,578][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:47,579][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:52,580][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:02:57,581][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:02,582][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:07,584][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:12,585][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:17,586][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:22,587][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:27,588][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:32,589][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:37,590][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:42,594][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:47,595][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:52,596][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:03:57,597][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:02,598][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:07,599][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:12,600][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:17,602][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:22,603][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:27,604][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:32,605][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:37,606][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:42,609][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:47,610][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:52,611][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:04:57,613][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:02,615][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:07,616][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:12,617][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:17,618][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:22,619][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:27,620][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:32,624][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:37,625][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:42,626][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:47,627][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:52,628][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:05:57,629][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:02,630][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:07,631][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:12,633][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:17,634][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:22,635][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:27,637][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:32,638][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:37,639][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:42,640][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:47,641][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:52,642][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:06:57,643][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:02,644][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:07,645][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:12,646][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:17,647][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:22,648][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:27,649][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:32,651][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:37,653][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:42,654][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:47,655][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:52,656][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:07:57,657][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:02,658][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:07,660][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:12,661][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:17,662][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:22,663][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:27,664][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:32,665][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:37,666][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:42,667][ INFO][PID:1396236] Checking for cancel request [2025-07-24 06:08:47,674][ INFO][PID:1396236] Periodic builder liveness probe: dead [2025-07-24 06:09:47,813][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.240 port 22 timed out [2025-07-24 06:10:57,958][ ERROR][PID:1396236] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.240 port 22 timed out [2025-07-24 06:12:08,103][ ERROR][PID:1396236] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.240 port 22 timed out [2025-07-24 06:13:18,174][ ERROR][PID:1396236] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.240 port 22: Connection timed out [2025-07-24 06:13:28,145][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 06:13:28,175][ ERROR][PID:1396236] Can't ssh to cancel build. [2025-07-24 06:13:38,146][WARNING][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-24 06:13:48,147][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 2 [2025-07-24 06:13:48,149][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.240 copr-rpmbuild-log [2025-07-24 06:14:48,217][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 06:14:58,219][WARNING][PID:1396236] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-24 06:15:08,219][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 3 [2025-07-24 06:15:08,221][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.240 copr-rpmbuild-log [2025-07-24 06:16:08,288][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 06:16:18,294][WARNING][PID:1396236] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-24 06:16:28,296][ INFO][PID:1396236] Downloading the builder-live.log file, attempt 4 [2025-07-24 06:16:28,297][ INFO][PID:1396236] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.240 copr-rpmbuild-log [2025-07-24 06:17:28,365][ ERROR][PID:1396236] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-24 06:17:38,369][ ERROR][PID:1396236] Giving up for unstable SSH, failures: 1753337618.146555, 1753337698.2191002, 1753337778.2944689, 1753337858.3694303 [2025-07-24 06:17:38,370][ ERROR][PID:1396236] Re-try request for task on 'ResallocHost, ticket_id=4086917, hostname=140.211.168.240, name=copr_osuosl_p09_prod_05778202_20250724_053751, requested_tags=['copr_builder', 'arch_ppc64le', 'arch_power9']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-07-24 06:17:38,371][ INFO][PID:1396236] Releasing VM back to pool [2025-07-24 06:17:38,380][ ERROR][PID:1396236] Backend process error: Three host tried without success: {'140.211.168.240', '140.211.168.248', '140.211.168.80'} [2025-07-24 06:17:38,381][WARNING][PID:1396236] Switching not-finished job state to 'failed' [2025-07-24 06:17:38,381][ INFO][PID:1396236] Worker failed build, took 1887.4408214092255 [2025-07-24 06:17:38,382][ INFO][PID:1396236] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "rpmsoftwaremanagement", "project_name": "dnf-nightly", "project_dirname": "dnf-nightly", "submitter": "rpmsoftwaremanagement", "ended_on": 1753337858.381412, "started_on": 1753335970.9405906, "submitted_on": null, "status": 0, "chroot": "fedora-eln-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "9316868-fedora-eln-ppc64le", "build_id": 9316868, "package_name": "microdnf", "package_version": "3.10.1-20250724012027.1.g26c9397", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rpmsoftwaremanagement/dnf-nightly/microdnf", "git_hash": "88f88d7940766cd57f4a5518e6fd19d3aa10f357", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rpmsoftwaremanagement/dnf-nightly--rpmsoftwaremanagement", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://kojipkgs.fedoraproject.org/repos/eln-build/latest/$basearch", "id": "https_kojipkgs_fedoraproject_org_repos_eln_build_latest_basearch", "name": "Additional repo https_kojipkgs_fedoraproject_org_repos_eln_build_latest_basearch" } ], "background": false, "bootstrap": "off", "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_ppc64le", "arch_power9" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf-nightly", "result_dir": "09316868-microdnf", "built_packages": "", "id": 9316868, "mockchain_macros": { "copr_username": "rpmsoftwaremanagement", "copr_projectname": "dnf-nightly", "vendor": "Fedora Project COPR (rpmsoftwaremanagement/dnf-nightly)" } } ] } [2025-07-24 06:17:38,415][ INFO][PID:1396236] Sending fedora-messaging bus message in build.end [2025-07-24 06:17:38,443][ INFO][PID:1396236] Compressing /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09316868-microdnf/builder-live.log by gzip [2025-07-24 06:17:38,444][ INFO][PID:1396236] Running command 'gzip /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09316868-microdnf/builder-live.log' as PID 1829785 [2025-07-24 06:17:38,445][ INFO][PID:1396236] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09316868-microdnf/builder-live.log) [2025-07-24 06:17:38,446][ INFO][PID:1396236] Compressing /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09316868-microdnf/backend.log by gzip [2025-07-24 06:17:38,447][ INFO][PID:1396236] Running command 'gzip /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf-nightly/fedora-eln-ppc64le/09316868-microdnf/backend.log' as PID 1829786