From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 7268555392064421888 X-Received: by 2002:a2e:9803:0:b0:2b6:e2c1:6cda with SMTP id a3-20020a2e9803000000b002b6e2c16cdamr969041ljj.46.1692342431781; Fri, 18 Aug 2023 00:07:11 -0700 (PDT) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a05:651c:a05:b0:2b9:5450:e122 with SMTP id k5-20020a05651c0a0500b002b95450e122ls448568ljq.0.-pod-prod-03-eu; Fri, 18 Aug 2023 00:07:09 -0700 (PDT) X-Google-Smtp-Source: AGHT+IGArkgLS9NpprDipOkjznG30eki2CeUu8Hhbk5lzzW+pmMELfDUshswAEET2kLUdyJa+QEC X-Received: by 2002:a2e:8e88:0:b0:2b6:a827:164f with SMTP id z8-20020a2e8e88000000b002b6a827164fmr1070298ljk.10.1692342429636; Fri, 18 Aug 2023 00:07:09 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1692342429; cv=none; d=google.com; s=arc-20160816; b=NdDIf2uvXHIvHETxBgkJgXHzQW2+1rOZmyB+YfzgJ+QbPPiJwC5zmyiTZhxQUlbXgP 0f7xlMM9LngVkc3vORZDH1ktO8bhQny5JHOQL1arpOHJke19RP0cwvZuRhoWsjerU7aL hq9KUyE69H+R+gSZH4XLgmPebHOPU+JXeQCLtTBROsjmL2U4GTAnP/Y8QROYQCNkCsJp Kk9Pnbfay+1j+HnRaxzzhwZZ119kyojCY0fzfrD175SFVxexUg0EOPCTNRluQQzvpcoe WR7BNi5gSBpLGi7H4L9Z8zenM6s/BuPa2z/82b7jSbux87srD3E4BAL/1nR0fOtz6Swd 3VMA== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from; bh=G41U+p5vMcmI1OQfHbGaMfqJm/yEJCadAssM5eWFnmM=; fh=swRs1+OhWjovx1IrvOlKBWsdq9manB5qdcuGoHoh5Ak=; b=ObucDm4Xw6F7hoYG+T2wqEz0u1EYt3vDXLnJH0VcLVlcfZvQWwRdxnLgWUwvRdEw1e dwF7gs+K4yHGv1FZrV/EHc/j+TNFiAO7WzF+YZp14YMTI0hQYF/SkePKPNw5fdUYQYG3 GYzo1tAtL0OgupaX0GkJ54k0y+sBoSOt1C7HtaAkgqkQyIVA1kl77UcaTZiZs+vffH2W drgunKdLpgBeMdv30XNGnFVpOCimTVPI8PwwPQdZKO4bauJyGhLnWOaAOb2BVv6BahgW M7P3FSI3BRr4fLPPw9DPIlP2SSlssijhTcCeqQFOHQiZOt1cfNxCAbF1RMI/a5VBjSMt NjmQ== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of ubely@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=ubely@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id c13-20020a2ea78d000000b002b9d5a29ef7si81415ljf.4.2023.08.18.00.07.09 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Fri, 18 Aug 2023 00:07:09 -0700 (PDT) Received-SPF: pass (google.com: domain of ubely@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of ubely@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=ubely@ilbers.de Received: from baighyz.m.ilbers.de (host-80-81-17-52.static.customer.m-online.net [80.81.17.52]) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8+deb9u1) with ESMTPSA id 37I7769j024239 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT) for ; Fri, 18 Aug 2023 09:07:08 +0200 From: Uladzimir Bely To: isar-users@googlegroups.com Subject: [PATCH 3/8] cibuilder.py: Split vm_start function to smaller subfunctions Date: Fri, 18 Aug 2023 09:07:01 +0200 Message-Id: <20230818070706.27913-4-ubely@ilbers.de> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20230818070706.27913-1-ubely@ilbers.de> References: <20230818070706.27913-1-ubely@ilbers.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: U9uGD/nUL35m vm_start is a pretty large function that does the following: - starts machine in qemu; - parses machine output (e.g., serial console); - runs remote command over ssh, if requested; - kills qemu process. Splitting it to smaller sub-functions improves code readability and potentially allows to reuse these sub-functions results in different tests tests. Signed-off-by: Uladzimir Bely --- testsuite/cibuilder.py | 186 +++++++++++++++++++++++------------------ 1 file changed, 103 insertions(+), 83 deletions(-) diff --git a/testsuite/cibuilder.py b/testsuite/cibuilder.py index 80b3aa88..8f28d05a 100755 --- a/testsuite/cibuilder.py +++ b/testsuite/cibuilder.py @@ -346,47 +346,70 @@ class CIBuilder(Test): self.fail('No command to run specified') - def vm_start(self, arch='amd64', distro='buster', - enforce_pcbios=False, skip_modulecheck=False, - image='isar-image-base', cmd=None, script=None): - time_to_wait = self.params.get('time_to_wait', default=DEF_VM_TO_SEC) - - self.log.info('===================================================') - self.log.info('Running Isar VM boot test for (' + distro + '-' + arch + ')') - self.log.info('Remote command is ' + str(cmd)) - self.log.info('Remote script is ' + str(script)) - self.log.info('Isar build folder is: ' + self.build_dir) - self.log.info('===================================================') - - self.check_init() - + def vm_turn_on(self, arch='amd64', distro='buster', image='isar-image-base', + enforce_pcbios=False): logdir = '%s/vm_start' % self.build_dir if not os.path.exists(logdir): os.mkdir(logdir) prefix = '%s-vm_start_%s_%s_' % (time.strftime('%Y%m%d-%H%M%S'), distro, arch) - fd, output_file = tempfile.mkstemp(suffix='_log.txt', prefix=prefix, + fd, boot_log = tempfile.mkstemp(suffix='_log.txt', prefix=prefix, dir=logdir, text=True) - os.chmod(output_file, 0o644) + os.chmod(boot_log, 0o644) latest_link = '%s/vm_start_%s_%s_latest.txt' % (logdir, distro, arch) if os.path.exists(latest_link): os.unlink(latest_link) - os.symlink(os.path.basename(output_file), latest_link) + os.symlink(os.path.basename(boot_log), latest_link) cmdline = start_vm.format_qemu_cmdline(arch, self.build_dir, distro, image, - output_file, None, enforce_pcbios) + boot_log, None, enforce_pcbios) cmdline.insert(1, '-nographic') self.log.info('QEMU boot line:\n' + ' '.join(cmdline)) + self.log.info('QEMU boot log:\n' + boot_log) + + p1 = subprocess.Popen('exec ' + ' '.join(cmdline), shell=True, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + self.log.debug("Started VM with pid %s" % (p1.pid)) - login_prompt = b'isar login:' + return p1, cmdline, boot_log + + + def vm_wait_boot(self, p1, timeout): + login_prompt = b' login:' + + poller = select.poll() + poller.register(p1.stdout, select.POLLIN) + poller.register(p1.stderr, select.POLLIN) + + while time.time() < timeout and p1.poll() is None: + events = poller.poll(1000 * (timeout - time.time())) + for fd, event in events: + if event != select.POLLIN: + continue + if fd == p1.stdout.fileno(): + # Wait for the complete string if it is read in chunks + # like "i", "sar", " login:" + time.sleep(0.01) + data = os.read(fd, 1024) + if login_prompt in data: + self.log.debug('Got login prompt') + return 0 + if fd == p1.stderr.fileno(): + app_log.error(p1.stderr.readline().rstrip()) + + self.log.error("Didn't get login prompt") + return 1 + + + def vm_parse_output(self, boot_log, bb_output, skip_modulecheck): # the printk of recipes-kernel/example-module module_output = b'Just an example' resize_output = None - - bb_output = start_vm.get_bitbake_env(arch, distro, image).decode() image_fstypes = start_vm.get_bitbake_var(bb_output, 'IMAGE_FSTYPES') wks_file = start_vm.get_bitbake_var(bb_output, 'WKS_FILE') + # only the first type will be tested in start_vm.py if image_fstypes.split()[0] == 'wic': if wks_file: @@ -399,72 +422,69 @@ class CIBuilder(Test): resize_output = b'resized filesystem to' if "sdimage-efi-btrfs" in wks_file: resize_output = b': resize device ' + rc = 0 + if os.path.exists(boot_log) and os.path.getsize(boot_log) > 0: + with open(boot_log, "rb") as f1: + data = f1.read() + if (module_output in data or skip_modulecheck): + if resize_output and not resize_output in data: + rc = 1 + self.log.error("No resize output while expected") + else: + rc = 2 + self.log.error("No example module output while expected") + return rc - timeout = time.time() + int(time_to_wait) - p1 = subprocess.Popen('exec ' + ' '.join(cmdline), shell=True, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) + def vm_turn_off(self, p1): + if p1.poll() is None: + p1.kill() + p1.wait() - if cmd is not None or script is not None: - try: - user='ci' - host='localhost' - port = 22 - for arg in cmdline: - match = re.match(r".*hostfwd=tcp::(\d*).*", arg) - if match: - port = match.group(1) - break - - rc = self.remote_run(user, host, port, cmd, script, p1, timeout) - - finally: - if p1.poll() is None: - self.log.debug('Killing qemu...') - p1.kill() - p1.wait() + self.log.debug("Stopped VM with pid %s" % (p1.pid)) - if rc != 0: - self.fail('Log ' + output_file) - return + def vm_start(self, arch='amd64', distro='buster', + enforce_pcbios=False, skip_modulecheck=False, + image='isar-image-base', cmd=None, script=None): + time_to_wait = self.params.get('time_to_wait', default=DEF_VM_TO_SEC) - try: - poller = select.poll() - poller.register(p1.stdout, select.POLLIN) - poller.register(p1.stderr, select.POLLIN) - while time.time() < timeout and p1.poll() is None: - events = poller.poll(1000 * (timeout - time.time())) - for fd, event in events: - if event != select.POLLIN: - continue - if fd == p1.stdout.fileno(): - # Wait for the complete string if it is read in chunks - # like "i", "sar", " login:" - time.sleep(0.01) - data = os.read(fd, 1024) - if login_prompt in data: - raise CanBeFinished - if fd == p1.stderr.fileno(): - app_log.error(p1.stderr.readline().rstrip()) - except CanBeFinished: - self.log.debug('Got login prompt') - finally: - if p1.poll() is None: - p1.kill() - p1.wait() + self.log.info('===================================================') + self.log.info('Running Isar VM boot test for (' + distro + '-' + arch + ')') + self.log.info('Remote command is ' + str(cmd)) + self.log.info('Remote script is ' + str(script)) + self.log.info('Isar build folder is: ' + self.build_dir) + self.log.info('===================================================') - if os.path.exists(output_file) and os.path.getsize(output_file) > 0: - with open(output_file, "rb") as f1: - data = f1.read() - if (module_output in data or skip_modulecheck) \ - and login_prompt in data: - if resize_output: - if resize_output in data: - return - else: - return - app_log.error(data.decode(errors='replace')) - - self.fail('Log ' + output_file) + self.check_init() + + timeout = time.time() + int(time_to_wait) + + p1, cmdline, boot_log = self.vm_turn_on(arch, distro, image, enforce_pcbios) + + rc = self.vm_wait_boot(p1, timeout) + if rc != 0: + self.vm_turn_off(p1) + self.fail('Failed to boot qemu machine') + + if cmd is not None or script is not None: + user='ci' + host='localhost' + port = 22 + for arg in cmdline: + match = re.match(r".*hostfwd=tcp::(\d*).*", arg) + if match: + port = match.group(1) + break + rc = self.remote_run(user, host, port, cmd, script, p1, timeout) + if rc != 0: + self.vm_turn_off(p1) + self.fail('Failed to run test over ssh') + else: + bb_output = start_vm.get_bitbake_env(arch, distro, image).decode() + rc = self.vm_parse_output(boot_log, bb_output, skip_modulecheck) + if rc != 0: + self.vm_turn_off(p1) + self.fail('Failed to parse output') + + self.vm_turn_off(p1) -- 2.20.1