pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
917928eb08905efa0657b77ae53aed6dff00decf
odoo_source/setup/package.py
T
Christophe Monniez 917928eb08 [IMP] packaging: update windows packaging 
Up to Odoo 13.0, the windows setup is 32 bits and embeds an old
postgresql 9.5 32 bits version too.

With this commit, the postgresql server is no longer embedded in the
resulting setup file. Instead, if the user choose to install the
postgresql server, the 12.4 version is downloaded from the official site
(version 10.14 in the case of a 32 bits windows).

Also, the Python target achitecture is choosen based on the Windows
system on which Odoo is installed.

This imply changes on the KVM images used to build the installer:

    - The Windows system on the KVM as to be 64 bits.
    - A 32 bits WinPython version must be in `c:/odoobuild/WinPy32`
    - A 64 bits WinPyhton version must be in `c:/odoobuild/WinPy64`

A little bit of cleaning also comes with this commit to get rid of the
old `openerp` references.

closes odoo/odoo#57727

X-original-commit: c47d7c67066ff15602b16f18c4c3acebfd3a2755
Signed-off-by: Christophe Monniez (moc) <moc@odoo.com>
2020-09-15 11:35:47 +00:00

568 lines
23 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import argparse
import logging
import os
import pexpect
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import traceback
from xmlrpc import client as xmlrpclib
from glob import glob
#----------------------------------------------------------
# Utils
#----------------------------------------------------------
ROOTDIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
TSTAMP = time.strftime("%Y%m%d", time.gmtime())
exec(open(os.path.join(ROOTDIR, 'odoo', 'release.py'), 'rb').read())
VERSION = version.split('-')[0].replace('saas~', '')
GPGPASSPHRASE = os.getenv('GPGPASSPHRASE')
GPGID = os.getenv('GPGID')
DOCKERVERSION = VERSION.replace('+', '')
INSTALL_TIMEOUT = 600
DOCKERUSER = """
RUN mkdir /var/lib/odoo && \
groupadd -g %(group_id)s odoo && \
useradd -u %(user_id)s -g odoo odoo -d /var/lib/odoo && \
mkdir /data && \
chown odoo:odoo /var/lib/odoo /data
USER odoo
""" % {'group_id': os.getgid(), 'user_id': os.getuid()}
class OdooTestTimeoutError(Exception):
pass
class OdooTestError(Exception):
pass
def run_cmd(cmd, chdir=None, timeout=None):
logging.info("Running command %s", cmd)
return subprocess.run(cmd, cwd=chdir, timeout=timeout)
def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'):
time.sleep(5)
uid = xmlrpclib.ServerProxy('%s:%s/xmlrpc/2/common' % (addr, port)).authenticate(
dbname, 'admin', 'admin', {}
)
modules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/2/object' % (addr, port)).execute(
dbname, uid, 'admin', 'ir.module.module', 'search', [('state', '=', 'installed')]
)
if len(modules) > 1:
time.sleep(1)
toinstallmodules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/2/object' % (addr, port)).execute(
dbname, uid, 'admin', 'ir.module.module', 'search', [('state', '=', 'to install')]
)
if toinstallmodules:
logging.error("Package test: FAILED. Not able to install dependencies of base.")
raise OdooTestError("Installation of package failed")
else:
logging.info("Package test: successfuly installed %s modules" % len(modules))
else:
logging.error("Package test: FAILED. Not able to install base.")
raise OdooTestError("Package test: FAILED. Not able to install base.")
def publish(args, pub_type, extensions):
"""Publish builded package (move builded files and generate a symlink to the latests)
:args: parsed program args
:pub_type: one of [deb, rpm, src, exe]
:extensions: list of extensions to publish
:returns: published files
"""
def _publish(release):
build_path = os.path.join(args.build_dir, release)
filename = release.split(os.path.sep)[-1]
release_dir = os.path.join(args.pub, pub_type)
release_path = os.path.join(release_dir, filename)
os.renames(build_path, release_path)
# Latest/symlink handler
release_abspath = os.path.abspath(release_path)
latest_abspath = release_abspath.replace(TSTAMP, 'latest')
if os.path.islink(latest_abspath):
os.unlink(latest_abspath)
os.symlink(release_abspath, latest_abspath)
return release_path
published = []
for extension in extensions:
release = glob("%s/odoo_*.%s" % (args.build_dir, extension))
if release:
published.append(_publish(release[0]))
return published
# ---------------------------------------------------------
# Generates Packages, Sources and Release files of debian package
# ---------------------------------------------------------
def gen_deb_package(args, published_files):
# Executes command to produce file_name in path, and moves it to args.pub/deb
def _gen_file(args, command, file_name, path):
cur_tmp_file_path = os.path.join(path, file_name)
with open(cur_tmp_file_path, 'w') as out:
subprocess.call(command, stdout=out, cwd=path)
shutil.copy(cur_tmp_file_path, os.path.join(args.pub, 'deb', file_name))
# Copy files to a temp directory (required because the working directory must contain only the
# files of the last release)
temp_path = tempfile.mkdtemp(suffix='debPackages')
for pub_file_path in published_files:
shutil.copy(pub_file_path, temp_path)
commands = [
(['dpkg-scanpackages', '--multiversion', '.'], "Packages"), # Generate Packages file
(['dpkg-scansources', '.'], "Sources"), # Generate Sources file
(['apt-ftparchive', 'release', '.'], "Release") # Generate Release file
]
# Generate files
for command in commands:
_gen_file(args, command[0], command[-1], temp_path)
# Remove temp directory
shutil.rmtree(temp_path)
if args.sign:
# Generate Release.gpg (= signed Release)
# Options -abs: -a (Create ASCII armored output), -b (Make a detach signature), -s (Make a signature)
subprocess.call(['gpg', '--default-key', GPGID, '--passphrase', GPGPASSPHRASE, '--yes', '-abs', '--no-tty', '-o', 'Release.gpg', 'Release'], cwd=os.path.join(args.pub, 'deb'))
# ---------------------------------------------------------
# Generates an RPM repo
# ---------------------------------------------------------
def gen_rpm_repo(args, file_name):
"""Genereate a rpm repo in publish directory"""
# Sign the RPM
rpmsign = pexpect.spawn('/bin/bash', ['-c', 'rpm --resign %s' % file_name], cwd=os.path.join(args.pub, 'rpm'))
rpmsign.expect_exact('Enter pass phrase: ')
rpmsign.send(GPGPASSPHRASE + '\r\n')
rpmsign.expect(pexpect.EOF)
# Removes the old repodata
shutil.rmtree(os.path.join(args.pub, 'rpm', 'repodata'))
# Copy files to a temp directory (required because the working directory must contain only the
# files of the last release)
temp_path = tempfile.mkdtemp(suffix='rpmPackages')
shutil.copy(file_name, temp_path)
run_cmd(['createrepo', temp_path]).check_returncode() # creates a repodata folder in temp_path
shutil.copytree(os.path.join(temp_path, "repodata"), os.path.join(args.pub, 'rpm', 'repodata'))
# Remove temp directory
shutil.rmtree(temp_path)
def _prepare_build_dir(args, win32=False):
"""Copy files to the build directory"""
logging.info('Preparing build dir "%s"', args.build_dir)
cmd = ['rsync', '-a', '--delete', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
if win32 is False:
cmd += ['--exclude', 'setup/win32']
run_cmd(cmd + ['%s/' % args.odoo_dir, args.build_dir])
for addon_path in glob(os.path.join(args.build_dir, 'addons/*')):
if args.blacklist is None or os.path.basename(addon_path) not in args.blacklist:
try:
shutil.move(addon_path, os.path.join(args.build_dir, 'odoo/addons'))
except shutil.Error as e:
logging.warning("Warning '%s' while moving addon '%s", e, addon_path)
if addon_path.startswith(args.build_dir) and os.path.isdir(addon_path):
logging.info("Removing ''".format(addon_path))
try:
shutil.rmtree(addon_path)
except shutil.Error as rm_error:
logging.warning("Cannot remove '{}': {}".format(addon_path, rm_error))
# Docker stuffs
class Docker():
"""Base Docker class. Must be inherited by specific Docker builder class"""
arch = None
def __init__(self, args):
"""
:param args: argparse parsed arguments
"""
self.args = args
self.tag = 'odoo-%s-%s-nightly-tests' % (DOCKERVERSION, self.arch)
self.container_name = None
self.exposed_port = None
dockerfiles = {
'tgz': os.path.join(args.build_dir, 'setup/package.dfsrc'),
'deb': os.path.join(args.build_dir, 'setup/package.dfdebian'),
'rpm': os.path.join(args.build_dir, 'setup/package.dffedora'),
}
self.dockerfile = dockerfiles[self.arch]
self.test_log_file = '/data/src/test-%s.log' % self.arch
self.build_image()
def build_image(self):
"""Build the dockerimage by copying Dockerfile into build_dir/docker"""
docker_dir = os.path.join(self.args.build_dir, 'docker')
docker_file_path = os.path.join(docker_dir, 'Dockerfile')
os.mkdir(docker_dir)
shutil.copy(self.dockerfile, docker_file_path)
with open(docker_file_path, 'a') as dockerfile:
dockerfile.write(DOCKERUSER)
shutil.copy(os.path.join(self.args.build_dir, 'requirements.txt'), docker_dir)
run_cmd(["docker", "build", "--rm=True", "-t", self.tag, "."], chdir=docker_dir, timeout=1200).check_returncode()
shutil.rmtree(docker_dir)
def run(self, cmd, build_dir, container_name, user='odoo', exposed_port=None, detach=False, timeout=None):
self.container_name = container_name
docker_cmd = [
"docker",
"run",
"--user=%s" % user,
"--name=%s" % container_name,
"--rm",
"--volume=%s:/data/src" % build_dir
]
if exposed_port:
docker_cmd.extend(['-p', '127.0.0.1:%s:%s' % (exposed_port, exposed_port)])
self.exposed_port = exposed_port
if detach:
docker_cmd.append('-d')
# preserve logs in case of detached docker container
cmd = '(%s) > %s 2>&1' % (cmd, self.test_log_file)
docker_cmd.extend([
self.tag,
"/bin/bash",
"-c",
"cd /data/src && %s" % cmd
])
run_cmd(docker_cmd, timeout=timeout).check_returncode()
def is_running(self):
dinspect = subprocess.run(['docker', 'container', 'inspect', self.container_name], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
return True if dinspect.returncode == 0 else False
def stop(self):
run_cmd(["docker", "stop", self.container_name]).check_returncode()
def test_odoo(self):
logging.info('Starting to test Odoo install test')
start_time = time.time()
while self.is_running() and (time.time() - start_time) < INSTALL_TIMEOUT:
time.sleep(5)
if os.path.exists(os.path.join(args.build_dir, 'odoo.pid')):
try:
_rpc_count_modules(port=self.exposed_port)
finally:
self.stop()
return
if self.is_running():
self.stop()
raise OdooTestTimeoutError('Odoo pid file never appeared after %s sec' % INSTALL_TIMEOUT)
raise OdooTestError('Error while installing/starting Odoo after %s sec.\nSee testlogs.txt in build dir' % int(time.time() - start_time))
def build(self):
"""To be overriden by specific builder"""
pass
def start_test(self):
"""To be overriden by specific builder"""
pass
class DockerTgz(Docker):
"""Docker class to build python src package"""
arch = 'tgz'
def build(self):
logging.info('Start building python tgz package')
self.run('python3 setup.py sdist --quiet --formats=gztar,zip', self.args.build_dir, 'odoo-src-build-%s' % TSTAMP)
os.rename(glob('%s/dist/odoo-*.tar.gz' % self.args.build_dir)[0], '%s/odoo_%s.%s.tar.gz' % (self.args.build_dir, VERSION, TSTAMP))
os.rename(glob('%s/dist/odoo-*.zip' % self.args.build_dir)[0], '%s/odoo_%s.%s.zip' % (self.args.build_dir, VERSION, TSTAMP))
logging.info('Finished building python tgz package')
def start_test(self):
if not self.args.test:
return
logging.info('Start testing python tgz package')
cmds = [
'service postgresql start',
'pip3 install /data/src/odoo_%s.%s.tar.gz' % (VERSION, TSTAMP),
'su postgres -s /bin/bash -c "createuser -s odoo"',
'su postgres -s /bin/bash -c "createdb mycompany"',
'su odoo -s /bin/bash -c "odoo -d mycompany -i base --stop-after-init"',
'su odoo -s /bin/bash -c "odoo -d mycompany --pidfile=/data/src/odoo.pid"',
]
self.run(' && '.join(cmds), self.args.build_dir, 'odoo-src-test-%s' % TSTAMP, user='root', detach=True, exposed_port=8069, timeout=300)
self.test_odoo()
logging.info('Finished testing tgz package')
class DockerDeb(Docker):
"""Docker class to build debian package"""
arch = 'deb'
def build(self):
logging.info('Start building debian package')
# Append timestamp to version for the .dsc to refer the right .tar.gz
cmds = ["sed -i '1s/^.*$/odoo (%s.%s) stable; urgency=low/' debian/changelog" % (VERSION, TSTAMP)]
cmds.append('dpkg-buildpackage -rfakeroot -uc -us -tc')
# As the packages are built in the parent of the buildir, we move them back to build_dir
cmds.append('mv ../odoo_* ./')
self.run(' && '.join(cmds), self.args.build_dir, 'odoo-deb-build-%s' % TSTAMP)
logging.info('Finished building debian package')
def start_test(self):
if not self.args.test:
return
logging.info('Start testing debian package')
cmds = [
'service postgresql start',
'su postgres -s /bin/bash -c "createdb mycompany"',
'/usr/bin/apt-get update -y',
'/usr/bin/dpkg -i /data/src/odoo_%s.%s_all.deb ; /usr/bin/apt-get install -f -y' % (VERSION, TSTAMP),
'su odoo -s /bin/bash -c "odoo -d mycompany -i base --stop-after-init"',
'su odoo -s /bin/bash -c "odoo -d mycompany --pidfile=/data/src/odoo.pid"',
]
self.run(' && '.join(cmds), self.args.build_dir, 'odoo-deb-test-%s' % TSTAMP, user='root', detach=True, exposed_port=8069, timeout=300)
self.test_odoo()
logging.info('Finished testing debian package')
class DockerRpm(Docker):
"""Docker class to build rpm package"""
arch = 'rpm'
def build(self):
logging.info('Start building fedora rpm package')
self.run('python3 setup.py --quiet bdist_rpm', self.args.build_dir, 'odoo-rpm-build-%s' % TSTAMP)
os.rename(glob('%s/dist/odoo-*.noarch.rpm' % self.args.build_dir)[0], '%s/odoo_%s.%s.rpm' % (self.args.build_dir, VERSION, TSTAMP))
logging.info('Finished building fedora rpm package')
def start_test(self):
if not self.args.test:
return
logging.info('Start testing rpm package')
cmds = [
'su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"',
'sleep 5',
'su postgres -c "createdb mycompany"',
'dnf install -d 0 -e 0 /data/src/odoo_%s.%s.rpm -y' % (VERSION, TSTAMP),
'su odoo -s /bin/bash -c "odoo -c /etc/odoo/odoo.conf -d mycompany -i base --stop-after-init"',
'su odoo -s /bin/bash -c "odoo -c /etc/odoo/odoo.conf -d mycompany --pidfile=/data/src/odoo.pid"',
]
self.run(' && '.join(cmds), args.build_dir, 'odoo-rpm-test-%s' % TSTAMP, user='root', detach=True, exposed_port=8069, timeout=300)
self.test_odoo()
logging.info('Finished testing rpm package')
# KVM stuffs
class KVM(object):
def __init__(self, args):
self.args = args
self.image = args.vm_winxp_image
self.ssh_key = args.vm_winxp_ssh_key
self.login = args.vm_winxp_login
def timeout(self, signum, frame):
logging.warning("vm timeout kill (pid: {})".format(self.kvm_proc.pid))
self.kvm_proc.terminate()
def start(self):
kvm_cmd = [
"kvm",
"-cpu", "Skylake-Client,hypervisor=on,hle=off,rtm=off",
"-smp", "2,sockets=2,cores=1,threads=1",
"-net", "nic,model=e1000e,macaddr=52:54:00:d3:38:5e",
"-net", "user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432",
"-m", "2048",
"-drive", "file=%s,snapshot=on" % self.image,
"-nographic",
"-serial", "none",
]
logging.info("Starting kvm: {}".format(" ".join(kvm_cmd)))
self.kvm_proc = subprocess.Popen(kvm_cmd)
time.sleep(60) # give some time to the VM to start, otherwise the SSH server may not be ready
signal.alarm(2400)
signal.signal(signal.SIGALRM, self.timeout)
try:
self.run()
finally:
signal.signal(signal.SIGALRM, signal.SIG_DFL)
self.kvm_proc.terminate()
time.sleep(10)
def ssh(self, cmd):
run_cmd([
'ssh',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-p', '10022',
'-i', self.ssh_key,
'%s@127.0.0.1' % self.login,
cmd
]).check_returncode()
def rsync(self, rsync_args, options=['--delete', '--exclude', '.git', '--exclude', '.tx', '--exclude', '__pycache__']):
cmd = [
'rsync',
'-a',
'-e', 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s' % self.ssh_key
]
cmd.extend(options)
cmd.extend(rsync_args)
run_cmd(cmd).check_returncode()
def run(self):
pass
class KVMWinBuildExe(KVM):
def run(self):
logging.info('Start building Windows package')
with open(os.path.join(self.args.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
f.write("VERSION=%s.%s\n" % (VERSION.replace('~', '_').replace('+', ''), TSTAMP))
with open(os.path.join(self.args.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
f.write("PYTHON_VERSION=%s\n" % self.args.vm_winxp_python_version)
with open(os.path.join(self.args.build_dir, 'setup/win32/Makefile.servicename'), 'w') as f:
f.write("SERVICENAME=%s\n" % nt_service_name)
remote_build_dir = '/cygdrive/c/odoobuild/server/'
self.ssh("mkdir -p build")
logging.info("Syncing Odoo files to virtual machine...")
self.rsync(['%s/' % self.args.build_dir, '%s@127.0.0.1:%s' % (self.login, remote_build_dir)])
self.ssh("cd {}setup/win32;time make allinone;".format(remote_build_dir))
self.rsync(['%s@127.0.0.1:%ssetup/win32/release/' % (self.login, remote_build_dir), '%s/' % self.args.build_dir])
logging.info('Finished building Windows package')
class KVMWinTestExe(KVM):
def run(self):
logging.info('Start testing Windows package')
setup_path = glob("%s/odoo_setup_*.exe" % self.args.build_dir)[0]
setupfile = setup_path.split('/')[-1]
setupversion = setupfile.split('odoo_setup_')[1].split('.exe')[0]
self.rsync(['%s' % setup_path, '%s@127.0.0.1:' % self.login])
self.ssh("TEMP=/tmp ./%s /S" % setupfile)
self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"Odoo %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion)
self.ssh('netsh advfirewall set publicprofile state off')
self.ssh('/cygdrive/c/"Program Files"/"Odoo {sv}"/python/python.exe \'c:\\Program Files\\Odoo {sv}\\server\\odoo-bin\' -d mycompany -i base --stop-after-init'.format(sv=setupversion))
_rpc_count_modules(port=18069)
logging.info('Finished testing Windows package')
def build_exe(args):
KVMWinBuildExe(args).start()
def test_exe(args):
if args.test:
KVMWinTestExe(args).start()
def parse_args():
ap = argparse.ArgumentParser()
build_dir = "%s-%s" % (ROOTDIR, TSTAMP)
log_levels = {"debug": logging.DEBUG, "info": logging.INFO, "warning": logging.WARN, "error": logging.ERROR, "critical": logging.CRITICAL}
ap.add_argument("-b", "--build-dir", default=build_dir, help="build directory (%(default)s)", metavar="DIR")
ap.add_argument("-p", "--pub", default=None, help="pub directory %(default)s", metavar="DIR")
ap.add_argument("--logging", action="store", choices=list(log_levels.keys()), default="info", help="Logging level")
ap.add_argument("--build-deb", action="store_true")
ap.add_argument("--build-rpm", action="store_true")
ap.add_argument("--build-tgz", action="store_true")
ap.add_argument("--build-win", action="store_true")
# Windows VM
ap.add_argument("--vm-winxp-image", default='/home/odoo/vm/win1036/win10_winpy36.qcow2', help="%(default)s")
ap.add_argument("--vm-winxp-ssh-key", default='/home/odoo/vm/win1036/id_rsa', help="%(default)s")
ap.add_argument("--vm-winxp-login", default='Naresh', help="Windows login %(default)s")
ap.add_argument("--vm-winxp-python-version", default='3.7.7', help="Windows Python version installed in the VM (default: %(default)s)")
ap.add_argument("-t", "--test", action="store_true", default=False, help="Test built packages")
ap.add_argument("-s", "--sign", action="store_true", default=False, help="Sign Debian package / generate Rpm repo")
ap.add_argument("--no-remove", action="store_true", help="don't remove build dir")
ap.add_argument("--blacklist", nargs="*", help="Modules to blacklist in package")
parsed_args = ap.parse_args()
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %I:%M:%S', level=log_levels[parsed_args.logging])
parsed_args.odoo_dir = ROOTDIR
return parsed_args
def main(args):
try:
if args.build_tgz:
_prepare_build_dir(args)
docker_tgz = DockerTgz(args)
docker_tgz.build()
try:
docker_tgz.start_test()
published_files = publish(args, 'tgz', ['tar.gz', 'zip'])
except Exception as e:
logging.error("Won't publish the tgz release.\n Exception: %s" % str(e))
if args.build_rpm:
_prepare_build_dir(args)
docker_rpm = DockerRpm(args)
docker_rpm.build()
try:
docker_rpm.start_test()
published_files = publish(args, 'rpm', ['rpm'])
if args.sign:
gen_rpm_repo(args, published_files[0])
except Exception as e:
logging.error("Won't publish the rpm release.\n Exception: %s" % str(e))
if args.build_deb:
_prepare_build_dir(args)
docker_deb = DockerDeb(args)
docker_deb.build()
try:
docker_deb.start_test()
published_files = publish(args, 'deb', ['deb', 'dsc', 'changes', 'tar.xz'])
gen_deb_package(args, published_files)
except Exception as e:
logging.error("Won't publish the deb release.\n Exception: %s" % str(e))
if args.build_win:
_prepare_build_dir(args, win32=True)
build_exe(args)
try:
test_exe(args)
published_files = publish(args, 'windows', ['exe'])
except Exception as e:
logging.error("Won't publish the exe release.\n Exception: %s" % str(e))
except Exception as e:
logging.error('Something bad happened ! : {}'.format(e))
traceback.print_exc()
finally:
if args.no_remove:
logging.info('Build dir "{}" not removed'.format(args.build_dir))
else:
if os.path.exists(args.build_dir):
shutil.rmtree(args.build_dir)
logging.info('Build dir %s removed' % args.build_dir)
if __name__ == '__main__':
args = parse_args()
if os.path.exists(args.build_dir):
logging.error('Build dir "%s" already exists.', args.build_dir)
sys.exit(1)
main(args)
Reference in New Issue View Git Blame Copy Permalink