#!/usr/bin/python3

import glob
import hashlib
import os
import shutil
import subprocess
import sys
import time
import traceback

from optparse import OptionParser
from rhn import rhnLockfile
from spacewalk.server import rhnSQL
from spacewalk.common.rhnLog import initLOG, log_time, log_clean
from spacewalk.common.rhnConfig import CFG, initCFG
from string import Template
from uyuni.common import usix, rhnLib, notificationUtils

sys.path.append("/usr/share/susemanager")

initCFG('server.susemanager')
rhnSQL.initDB()

basepath = CFG.MOUNT_POINT or '/var/spacewalk'
LOCK  = None
BETA  = None
logfile = '/var/log/rhn/mgr-create-bootstrap-repo/mgr-create-bootstrap-repo.log'
UYUNI = None

_sql_synced_proucts = rhnSQL.Statement("""
   SELECT sp.product_id id, spsr.root_product_id
     FROM suseProducts sp
     JOIN suseProductSCCRepository spsr ON spsr.product_id = sp.id
LEFT JOIN rhnChannel c ON spsr.channel_label = c.label
    WHERE spsr.mandatory = 'Y'
 GROUP BY sp.id, spsr.root_product_id
   HAVING COUNT(c.label) = COUNT(spsr.mandatory)
""")

_sql_find_root_channel_label = """
SELECT label
 FROM rhnChannel
WHERE id IN (
  SELECT DISTINCT c.parent_channel
    FROM rhnChannel c
    JOIN suseProductChannel pc ON pc.channel_id = c.id
    JOIN suseProducts p ON pc.product_id = p.id
   WHERE p.product_id IN ( {0} )
)
""";

_sql_filter_root_channel_label = """
select c.label
  from suseproducts sp
  join suseproductchannel spc on sp.id = spc.product_id
  join rhnchannel c on spc.channel_id = c.id
 where sp.product_id in ( {0} )
   and c.label in ( {1} );
""";

_sql_find_custom_parent_channel_labels = """
SELECT label
  FROM rhnChannel
 WHERE parent_channel IS NULL
   AND org_id IS NOT NULL
""";

_sql_find_pkgs_all = """
SELECT distinct
       pkg.id AS id,
       pn.name || '-' || evr_t_as_vre_simple(pevr.evr) || '.' || pa.label AS nvrea,
       pa.label AS arch,
       pkg.path
   FROM rhnPackage pkg
   JOIN rhnPackageArch pa ON pkg.package_arch_id = pa.id
   JOIN rhnPackageName pn ON pkg.name_id = pn.id
   JOIN rhnPackageEVR pevr ON pkg.evr_id = pevr.id
   JOIN suseChannelPackageRetractedStatusView CP ON CP.package_id = pkg.id
   JOIN rhnChannel c ON CP.channel_id = c.id
   JOIN (
          SELECT I_C.*
            FROM suseProducts I_SP
            JOIN suseProductChannel I_PC ON I_SP.id = I_PC.product_id
            JOIN rhnChannel I_C ON I_PC.channel_id = I_C.id
           WHERE  I_SP.product_id IN ( %s )
           UNION
           SELECT *
             FROM rhnChannel
            WHERE org_id IS NOT NULL
              AND parent_channel IN (
                                     SELECT pc.id
                                       FROM rhnChannel pc
                                      WHERE pc.label = :parentchannel)
        ) spc ON spc.id = c.id
  WHERE pn.name = :pkgname
    AND NOT CP.is_retracted
ORDER BY pkg.id
"""

_sql_find_pkgs = """
SELECT distinct
        pkg.id AS id,
        PN.name || '-' || evr_t_as_vre_simple(full_list.evr) || '.' || full_list.arch_label AS nvrea,
        full_list.arch_label AS arch,
        pkg.path
  FROM  (
         SELECT  I_P.name_id name_id,
                 MAX(I_PE.evr) evr,
                 I_PA.id AS arch_id,
                 I_PA.label AS arch_label,
                 (CASE WHEN channels.parent_channel IS NULL THEN channels.id ELSE channels.parent_channel END) AS root
           FROM  (
                  SELECT I_C.*
                   FROM suseProducts I_SP
                   JOIN suseProductChannel I_PC ON I_SP.id = I_PC.product_id
                   JOIN rhnChannel I_C ON I_PC.channel_id = I_C.id
                  WHERE  I_SP.product_id IN ( %s )
                  UNION
                  SELECT *
                    FROM rhnChannel
                   WHERE org_id IS NOT NULL
                     AND parent_channel IN (
                                            SELECT pc.id
                                              FROM rhnChannel pc
                                             WHERE pc.label = :parentchannel)
                  ) channels
           JOIN  rhnChannelNewestPackage I_CNP ON channels.id = I_CNP.channel_id
           JOIN  rhnPackage I_P ON I_CNP.package_id = I_P.id
           JOIN  rhnPackageEVR I_PE ON I_P.evr_id = I_PE.id
           JOIN  rhnPackageArch I_PA ON I_P.package_arch_id = I_PA.id
       GROUP BY  I_P.name_id, I_PA.label, I_PA.id, root
     ) full_list,
       rhnPackage pkg
       JOIN rhnPackageName pn ON pkg.name_id = pn.id
       JOIN rhnPackageEVR pevr ON pkg.evr_id = pevr.id
       JOIN rhnChannelPackage CP ON CP.package_id = pkg.id
       JOIN rhnChannel c ON CP.channel_id = c.id
 WHERE full_list.name_id = pkg.name_id
   AND full_list.evr = pevr.evr
   AND full_list.arch_id = pkg.package_arch_id
   AND (c.parent_channel = full_list.root OR c.id = full_list.root)
   AND pn.name = :pkgname
ORDER BY pkg.id
""";

_sql_find_pkgs_custom = """
SELECT DISTINCT
        pkg.id AS id,
        PN.name || '-' || evr_t_as_vre_simple(full_list.evr) || '.' || full_list.arch_label AS nvrea,
        full_list.arch_label AS arch,
        pkg.path
  FROM  (
         SELECT  I_P.name_id name_id,
                 MAX(I_PE.evr) evr,
                 I_PA.id AS arch_id,
                 I_PA.label AS arch_label,
                 (CASE WHEN channels.parent_channel IS NULL THEN channels.id ELSE channels.parent_channel END) AS root
           FROM  (
                  SELECT *
                    FROM rhnChannel
                   WHERE org_id IS NOT NULL
                     AND label = :parentchannel
                      OR parent_channel IN (
                                            SELECT pc.id
                                              FROM rhnChannel pc
                                             WHERE pc.label = :parentchannel)
                  ) channels
           JOIN  rhnChannelNewestPackage I_CNP ON channels.id = I_CNP.channel_id
           JOIN  rhnPackage I_P ON I_CNP.package_id = I_P.id
           JOIN  rhnPackageEVR I_PE ON I_P.evr_id = I_PE.id
           JOIN  rhnPackageArch I_PA ON I_P.package_arch_id = I_PA.id
       GROUP BY  I_P.name_id, I_PA.label, I_PA.id, root
     ) full_list,
       rhnPackage pkg
       JOIN rhnPackageName pn ON pkg.name_id = pn.id
       JOIN rhnPackageEVR pevr ON pkg.evr_id = pevr.id
       JOIN rhnChannelPackage CP ON CP.package_id = pkg.id
       JOIN rhnChannel c ON CP.channel_id = c.id
 WHERE full_list.name_id = pkg.name_id
   AND full_list.evr = pevr.evr
   AND full_list.arch_id = pkg.package_arch_id
   AND (c.parent_channel = full_list.root OR c.id = full_list.root)
   AND pn.name = :pkgname
ORDER BY pkg.id
""";


_find_mand_modified_repos = """
select X.product_id, c0.label, c0.last_synced,
       (select 1 from dual where c0.last_synced > :filemod) as newer
from rhnChannel c0
join suseProductChannel spc ON spc.channel_id = c0.id
join suseProducts p ON spc.product_id = p.id
join (
     select sp.product_id , spsr.root_product_id
       from suseProducts sp
       join suseProductSCCRepository spsr ON spsr.product_id = sp.id
  left join rhnChannel c ON spsr.channel_label = c.label
      where spsr.mandatory = 'Y'
   group by sp.id, spsr.root_product_id
     having COUNT(c.label) = COUNT(spsr.mandatory)
   ) X on X.product_id = p.product_id
where X.product_id in ( %s )
""";

_find_modified_repos_by_basechannel = """
SELECT c.label, c.last_synced,
       (SELECT 1 FROM DUAL WHERE c.last_synced > :filemod) AS newer
  FROM rhnchannel c
 WHERE (SELECT id FROM rhnchannel WHERE label = :basechannel) IN (c.parent_channel, c.id);
""";

_lookupToolsProductIds = """
SELECT sp.product_id
  FROM suseProducts sp
  JOIN rhnChannelFamily cf ON sp.channel_family_id = cf.id
 WHERE cf.label = 'SLE-M-T';
""";

def releaseLOCK():
    global LOCK
    if LOCK:
        LOCK.release()
        LOCK = None


def log_error(msg):
    frame = traceback.extract_stack()[-2]
    log_clean(0, "{0}: {1}.{2}({3}) - {4}".format(log_time(), frame[0], frame[2], frame[1], msg))
    sys.stderr.write("{0}\n".format(msg))


def log(msg, level=0):
    frame = traceback.extract_stack()[-2]
    log_clean(level, "{0}: {1}.{2}({3}) - {4}".format(log_time(), frame[0], frame[2], frame[1], msg))
    if level < 1:
        sys.stdout.write("{0}\n".format(msg))

def isBeta():
    global BETA
    global UYUNI
    if BETA is None:
        initCFG('java')
        BETA = CFG.PRODUCT_TREE_TAG == 'Beta'
        UYUNI = CFG.PRODUCT_TREE_TAG == 'Uyuni'
        initCFG('server.susemanager')
    return BETA

def isUyuni():
    global BETA
    global UYUNI
    if UYUNI is None:
        initCFG('java')
        BETA = CFG.PRODUCT_TREE_TAG == 'Beta'
        UYUNI = CFG.PRODUCT_TREE_TAG == 'Uyuni'
        initCFG('server.susemanager')
    return UYUNI

def cli():

    usage = "usage: %prog [options] [additional_pkg1 additional_pkg2 ...]"
    parser = OptionParser(usage=usage, description=""
            "Tool to generate repositories containing the required software to "
            "register at SUSE Manager Server. Logs are written to "
            "/var/log/rhn/mgr-create-bootstrap-repo/mgr-create-bootstrap-repo.log .")

    parser.add_option('-n', '--dryrun', action='store_true', dest='dryrun',
                      help='Dry run. Show only changes - do not execute them')
    parser.add_option('-i', '--interactive', action='store_true', dest='interactive',
                      help='Interactive mode (default)')
    parser.add_option('-l', '--list', action='store_true', dest='list',
                      help='list available distributions')
    parser.add_option('-c', '--create', action='store', dest='create',
                      help='create bootstrap repo for given distribution label')
    parser.add_option('-a', '--auto', action='store_true', dest='auto',
                      help='Automatic Mode. Generate all available bootstrap repos')
    parser.add_option('', '--datamodule', action="store", dest='datamodule',
                      help='Use an own datamodule (Default: mgr_bootstrap_data)')
    parser.add_option('-d', '--debug', action='store_true', dest='debug',
                      help='Enable debug mode')


    parser.add_option('-f', '--flush', action='store_true', dest='flush',
                      help='when used in conjuction with --create, deletes the target repository before creating it (default)')
    parser.add_option('--no-flush', action='store_true', dest='noflush',
                      help='when used in conjuction with --create, prevent deletion of the target repository before creating it')
    parser.add_option('--force', action='store_true', dest='force',
                      help='Force creation even when not all required channels are available')
    parser.add_option('', '--with-custom-channels', action='store_true', dest='usecustomchannels',
                      help='Take custom channels into account when searching for newest package versions')
    parser.add_option('', '--with-parent-channel', action="store", dest='parentchannel',
                      help='use child channels below this parent')

    options, args = parser.parse_args()

    if options.debug:
        initLOG(logfile, level=5)
    else:
        initLOG(logfile, CFG.DEBUG or 1)
    flush = CFG.BOOTSTRAP_REPO_FLUSH
    if options.flush:
        flush = options.flush
    if options.noflush:
        flush = False
    options.flush = flush

    log(sys.argv, 1)
    if isBeta():
        log("Is a Beta installation", 1)

    blacklistedPids = []
    if isUyuni():
        log("Is a Uyuni installation", 1)
        blacklistedPids = list_products_needs_tools_subscription()

    modulename = options.datamodule or CFG.BOOTSTRAP_REPO_DATAMODULE
    try:
        bootstrap_data = __import__(modulename)
        for label in bootstrap_data.DATA:
            if 'PDID' in bootstrap_data.DATA[label]:
                if not isinstance(bootstrap_data.DATA[label]['PDID'], usix.ListType):
                    bootstrap_data.DATA[label]['PDID'] = list(map(str, [x for x in [int(bootstrap_data.DATA[label]['PDID'])] if x not in blacklistedPids]))
                else:
                    bootstrap_data.DATA[label]['PDID'] = list(map(str, [x for x in [int(pdid) for pdid in bootstrap_data.DATA[label]['PDID']] if x not in blacklistedPids]))
                if isBeta() and 'BETAPDID' in bootstrap_data.DATA[label]:
                    bootstrap_data.DATA[label]['PDID'].extend(list(map(str, [x for x in [int(pdid) for pdid in bootstrap_data.DATA[label]['BETAPDID']] if x not in blacklistedPids])))

    except ImportError as e:
        log_error("Unable to load module '%s'" % modulename)
        log_error(str(e))
        sys.exit(1)

    if not options.list and not options.create and not options.auto:
        options.interactive = True

    return options, args, bootstrap_data

def list_products_needs_tools_subscription():
    h = rhnSQL.Statement(_lookupToolsProductIds)
    return [x['product_id'] for x in rhnSQL.fetchall_dict(h) or []]


def find_root_channel_labels(pdids):
    """
    Get root channel labels for selected distribution

    :return: list of root channel labels
    """
    h = rhnSQL.Statement( _sql_find_root_channel_label.format(pdids))
    root_labels = [x['label'] for x in rhnSQL.fetchall_dict(h) or []]
    log("Root Labels: {}".format(root_labels), 3)
    if len(root_labels) > 1:
        h = rhnSQL.Statement( _sql_filter_root_channel_label.format(pdids, ', '.join(["'{}'".format(x) for x in root_labels])))
        filtered_root_labels = [x['label'] for x in rhnSQL.fetchall_dict(h) or []]
        log("Filtered Root Labels: {}".format(filtered_root_labels), 3)
        if len(filtered_root_labels) >= 1:
            return filtered_root_labels
        # else: if all are gone, the pids did not contain a base product.
        #       In this case we return all root labels found by the first query
    return root_labels


def find_custom_parent_channel_labels():
    """
    Get custom parent channel labels

    :return: list of custom parent channel labels
    """
    h = rhnSQL.Statement(_sql_find_custom_parent_channel_labels)
    return [x['label'] for x in rhnSQL.fetchall_dict(h) or []]


def list_labels(mgr_bootstrap_data, force=False, do_print=True):
    """
    Create list of labels and return a structure of them for the menu.

    :return:
    """
    label_map = {}
    synced_products = list(map(str, [x['id'] for x in rhnSQL.fetchall_dict(_sql_synced_proucts) or []]))
    custom_parent_channels = find_custom_parent_channel_labels()
    label_index = 1
    for label in sorted(mgr_bootstrap_data.DATA.keys()):
        if ('PDID' in mgr_bootstrap_data.DATA[label] and mgr_bootstrap_data.DATA[label]['PDID'] and
            (all(elem in synced_products for elem in mgr_bootstrap_data.DATA[label]['PDID']) or
                (force and any(elem in synced_products for elem in mgr_bootstrap_data.DATA[label]['PDID'])))):

            if label in ('RHEL8-x86_64', 'RHEL9-x86_64'):
                if not connected_to_rhel_cdn(find_root_channel_labels(', '.join(mgr_bootstrap_data.DATA[label]['PDID']))):
                    # skip native RHEL if not connected to cdn
                    log("{} not connected to CDN. Skipping".format(label), 1)
                    continue

            if do_print:
                print("{0}. {1}".format(label_index, label))
            label_map[label_index] = label
            label_index += 1
        elif 'BASECHANNEL' in mgr_bootstrap_data.DATA[label] and mgr_bootstrap_data.DATA[label]['BASECHANNEL'] in custom_parent_channels:
            if do_print:
                print("{0}. {1}".format(label_index, label))
            label_map[label_index] = label
            label_index += 1
    return label_map

def cleanup_dir(path):
    if os.path.exists(path):
        try:
            shutil.rmtree(path)
            log("REMOVE dir {0}".format(path), 3)
        except OSError as err:
            log_error('Error while deleting {0}: {1}'.format(path, err))
            return False
    return True


def create_repo(label, options, mgr_bootstrap_data, additional=[]):
    pdids = None
    usecustomchannels = options.usecustomchannels
    parentchannel = options.parentchannel

    if 'PDID' in mgr_bootstrap_data.DATA[label]:
        pdids = ', '.join(mgr_bootstrap_data.DATA[label]['PDID'])
        if label in ('RHEL8-x86_64', 'RHEL9-x86_64'):
            if not connected_to_rhel_cdn(find_root_channel_labels(pdids)):
                log("WARNING: {} not connected to CDN.".format(label))


        if (label.startswith('RES') or label.startswith('RHEL') or label.lower().startswith('ubuntu')):
            usecustomchannels = True
        if isUyuni():
            usecustomchannels = True
            for plabel in find_root_channel_labels(pdids):
                # we take the first one
                parentchannel = plabel
                break
    else:
        usecustomchannels = True
        parentchannel = mgr_bootstrap_data.DATA[label]['BASECHANNEL']

    destdir = os.path.normpath(mgr_bootstrap_data.DATA[label]['DEST'])
    dirprefix, lastdir = os.path.split(destdir)
    destdirtmp = os.path.join(dirprefix, "{0}.{1}".format(lastdir, "tmp"))
    destdirold = os.path.join(dirprefix, "{0}.{1}".format(lastdir, "old"))
    errors = 0
    messages = []
    suggestions = {
        'no-packages': None
    }

    if usecustomchannels:
        if pdids:
            root_labels = find_root_channel_labels(pdids)
        else:
            root_labels = find_custom_parent_channel_labels()
        if parentchannel and parentchannel not in root_labels:
            log_error("'{0}' not found in existing parent channel options '{1}'".format(parentchannel, root_labels))
            return 1
        elif not parentchannel:
            if len(root_labels) > 1:
                log_error("Multiple options for parent channel found. Please use option --with-parent-channel <label>")
                log_error("and choose one of:")
                for l in root_labels:
                    log_error("- {0}".format(l))
                return 1
            elif len(root_labels) == 1:
                parentchannel = root_labels[0]
            else:
                log("WARNING: no parent channel found. Execute without using custom channels")
                parentchannel = ""
    else:
        parentchannel = ""

    if not cleanup_dir(destdirtmp):
        return 1
    if not cleanup_dir(destdirold):
        return 1

    if options.dryrun:
        log("Create directory: {0}".format(destdirtmp))
    else:
        if not os.path.exists(destdir):
            os.makedirs(destdirtmp)
        else:
            log("Copy destdir to tempdir", 3)
            shutil.copytree(destdir, destdirtmp)

    print()
    if label.startswith('RES') or label.startswith('RHEL'):
        log("Creating bootstrap repo for latest Service Pack of {0}".format(label))
    else:
        log("Creating bootstrap repo for {0}".format(label))

    if pdids and not options.force:
        h = rhnSQL.prepare(rhnSQL.Statement(_sql_find_pkgs % (pdids)))
    elif pdids and options.force:
        h = rhnSQL.prepare(rhnSQL.Statement(_sql_find_pkgs_all % (pdids)))
    else:
        h = rhnSQL.prepare(rhnSQL.Statement(_sql_find_pkgs_custom ))
    packagelist = mgr_bootstrap_data.DATA[label]['PKGLIST']
    repotype = mgr_bootstrap_data.DATA[label].get('TYPE', 'yum')
    packagelist.extend(additional)
    log("The bootstrap repo should contain the following packages: {0}".format(packagelist), 2)

    debs_dir = os.path.join(destdirtmp, 'debs')
    if repotype == 'deb' and not options.dryrun:
        # reprepro is picky with packages having the same name, but different checksums.
        # To avoid aborting, we copy the old packages to debs and add or overwrite them
        # with new found packages and run reprepro again (bsc#1184330)
        if not os.path.exists(debs_dir):
            os.makedirs(debs_dir)
        for file_path in glob.glob(os.path.join(destdirtmp, 'pool/**/*.deb'), recursive=True):
            shutil.copy2(file_path, debs_dir)
        cleanup_dir(os.path.join(destdirtmp, 'pool'))
        cleanup_dir(os.path.join(destdirtmp, 'db'))
        cleanup_dir(os.path.join(destdirtmp, 'dists'))

    for pkgaltstr in packagelist:
        alt = pkgaltstr.split("|")
        altpretty =  ', '.join("'%s'" %(e) for e in alt)
        altcount = 0
        for pkgname in alt:
            optional = False
            if pkgname[-1] == "*":
                optional = True
                pkgname = pkgname[:-1]
            altcount += 1
            h.execute(parentchannel=parentchannel, pkgname=pkgname)
            pkgs = h.fetchall_dict() or []
            log("Package {0} found {1} resulting packages:".format(
                pkgname, len(pkgs)), 2)
            if len(pkgs) == 0:
                if optional:
                    log("Optional package '{0}' not found".format(pkgname))
                    continue
                if altcount >= len(alt):
                    if len(alt) > 1:
                        messages.append("ERROR: none of %s found" % altpretty)
                    else:
                        messages.append("ERROR: package '%s' not found" % pkgname)
                    errors += 1
                    if not suggestions['no-packages']:
                        suggestions['no-packages'] = (
                             'mgr-create-bootstrap-repo uses the locally synchronized versions of files\n' +
                             'from the Tools repository, and uses the locally synchronized pool channel\n' +
                             'for dependency resolution.\n' +
                             'Both should be fully synced before running the mgr-create-bootstrap-repo script.\n')
                continue
            for p in pkgs:
                log(p, 2)
                rpmdir = os.path.join(destdirtmp, p['arch']) if repotype != 'deb' else debs_dir
                if not os.path.exists(rpmdir) and not options.dryrun:
                    os.makedirs(rpmdir)
                log("copy '%s'" % p['nvrea'])
                log("copy {0} / {1} to {2}".format(basepath, p['path'], rpmdir), 2)
                if not options.dryrun:
                    shutil.copy2(os.path.join(basepath, p['path']), rpmdir)
            break
    if options.dryrun:
        if repotype == 'deb':
            debs = " ".join([os.path.join(debs_dir, f) for f in os.listdir(debs_dir)])
            log("/usr/bin/reprepro -b {0} includedeb {1} {2}".format(destdirtmp, "bootstrap", debs))
        else:
            log("createrepo -s sha %s" % destdirtmp)
    else:
        if repotype == 'deb':
            reprepro_conf_tmpl = Template("Origin: $origin\n" \
            "Label: $label\n" \
            "Codename: $codename\n" \
            "Architectures: $arches\n" \
            "Components: $comps\n" \
            "Description: $desc\n")
            codename="bootstrap"
            reprepro_conf = reprepro_conf_tmpl.substitute(origin="mgr",
                    label="mgr", codename=codename, arches="amd64 i386 armhf", comps="main", desc="Bootstrap repo")
            reprepro_conf_dir = os.path.join(destdirtmp, "conf")
            if not os.path.exists(reprepro_conf_dir):
                os.makedirs(reprepro_conf_dir)
            with open(os.path.join(reprepro_conf_dir, "distributions"), "w") as conf_file:
                conf_file.write(reprepro_conf)
            log("Created reprepro config file in {0}".format(os.path.join(destdirtmp, "distributions")), 2)
            debs = [os.path.join(debs_dir, f) for f in os.listdir(debs_dir)]
            try:
                subprocess.run(["/usr/bin/reprepro", "-b", destdirtmp, "includedeb", codename] + debs, check=True)
                log("Removing directory {0}".format(debs_dir), 2)
                shutil.rmtree(debs_dir, ignore_errors=True)
            except subprocess.CalledProcessError as err:
                log_error("Error creating bootstrap repo.")
                log(err, 2)
                return 1
        else:
            os.system("createrepo -s sha %s" % destdirtmp)
        # ensure venv-enabled-{ARCH}.txt doesn't exist in repo with no salt bundle package
        # create venv-enabled-{ARCH}.txt for repos with salt bundle package
        for file_path in glob.glob(os.path.join(destdirtmp, 'venv-enabled-*.txt')):
            os.remove(file_path)
        for file_path in sorted(glob.glob(os.path.join(destdirtmp, '**/venv-salt-minion*.*'), recursive=True)):
            rel_path = os.path.relpath(file_path, start=destdirtmp)
            (l_path, ext) = rel_path.rsplit('.', 1)
            if ext:
                dg = hashlib.sha256()
                with open(file_path, "rb") as pkg_fh:
                    while True:
                        buff = pkg_fh.read(0x1000)
                        if not buff:
                            break
                        dg.update(buff)
                (l_path, arch) = l_path.rsplit('.' if ext == 'rpm' else '_', 1)
                with open(os.path.join(destdirtmp, "venv-enabled-{}.txt".format(arch)), "w") as venv_enabled_file:
                    venv_enabled_file.write("{}  {}\n".format(dg.hexdigest(), rel_path))
                    venv_enabled_file.close()
        # move tmp dir to final location
        if os.path.exists(destdir):
            os.rename(destdir, destdirold)
        os.rename(destdirtmp, destdir)
        cleanup_dir(destdirold)

    if errors:
        for m in messages:
            log_error(m)
        if (label.startswith('RES') or label.startswith('RHEL') or label.lower().startswith('ubuntu'))  and not usecustomchannels:
            log_error("If the installation media was imported into a custom channel, try to run again with --with-custom-channels option")
        suggestions = list([_f for _f in list(suggestions.values()) if _f])
        if suggestions:
            log_error("\nSuggestions:")
            for suggestion in suggestions:
                log_error(suggestion)
        notificationUtils.CreateBootstrapRepoFailed(label, "\n".join(messages)).store()
        return 1
    return 0

def generate_repo_view(mgr_bootstrap_data):
    repos = {}
    for dist in sorted(list_labels(mgr_bootstrap_data, do_print=False).values()):
        if mgr_bootstrap_data.DATA[dist]['DEST'] not in repos:
            repos[mgr_bootstrap_data.DATA[dist]['DEST']] = {}
        repos[mgr_bootstrap_data.DATA[dist]['DEST']][dist] = mgr_bootstrap_data.DATA[dist]
    return repos

def connected_to_rhel_cdn(root_channel_labels):
    # only 1 entry in root_channel_labels expected
    if len(root_channel_labels) != 1:
        return False
    rcl = root_channel_labels.pop()
    _child_connected_to_redhat_cdn = """
    select ch.id
      from rhnchannel ch
      join rhnchannelcontentsource chcc on ch.id = chcc.channel_id
      join rhncontentsource cc on chcc.source_id = cc.id
     where ch.parent_channel in (select c.id
                                   from rhnchannel c
                                  where c.label = :parentlabel
                                    and c.parent_channel is NULL)
       and ch.org_id IS NOT NULL
       and (cc.source_url like '%cdn.redhat.com%'
            or LOWER(cc.source_url) like '%/baseos%'
            or LOWER(cc.source_url) like '%/appstream%');
    """
    h = rhnSQL.prepare(rhnSQL.Statement(_child_connected_to_redhat_cdn))
    h.execute(parentlabel=rcl)
    res = h.fetchall_dict() or False
    if not res:
        return False
    return True

def find_dists_for_regeneration(dest, dists, doall=False):
    regenerate = []
    for label, dist in dists.items():
        if label in ('RHEL8-x86_64', 'RHEL9-x86_64') and 'PDID' in dist:
            if not connected_to_rhel_cdn(find_root_channel_labels(', '.join(dist['PDID']))):
                # skip native RHEL if not connected to cdn
                log("{} not connected to CDN. Skipping".format(label), 1)
                continue
        destfile = os.path.join(dest, 'repodata', 'repomd.xml')
        if 'TYPE' in dist and dist['TYPE'] == 'deb':
            destfile = os.path.join(dest, 'dists', 'bootstrap', 'Release')

        filemodtime = 0
        if os.path.exists(destfile):
            filemodtime = os.path.getmtime(destfile)

        log("{0} modified: {1}".format(destfile, filemodtime), 2)
        if 'PDID' in dist:
            pdids = ', '.join(dist['PDID'])
            h = rhnSQL.prepare(rhnSQL.Statement(_find_mand_modified_repos % (pdids)))
            h.execute(filemod=time.strftime('%Y-%m-%d %H:%M:%S %z', time.localtime(filemodtime)))
        if 'BASECHANNEL' in dist:
            h = rhnSQL.prepare(rhnSQL.Statement(_find_modified_repos_by_basechannel))
            h.execute(filemod=time.strftime('%Y-%m-%d %H:%M:%S %z', time.localtime(filemodtime)), basechannel=dist['BASECHANNEL'])

        res = h.fetchall_dict() or []
        if not res:
            continue
        regen = True
        oneNewerTimestamp = 0
        for channelinfo in res:
            if doall and channelinfo['last_synced']:
                log("{0} available. Full regeneration requested".format(channelinfo['label']), 2)
                regen = regen and True
            elif not doall and channelinfo['newer'] == 1:
                log("{0} modified after last bootstrap generation".format(channelinfo['label']), 2)
                regen = regen and True
                if channelinfo['last_synced'] and channelinfo['last_synced'].timestamp() > oneNewerTimestamp:
                    oneNewerTimestamp = channelinfo['last_synced'].timestamp()
            else:
                log("{0} not modified".format(channelinfo['label']), 2)
                regen = regen and False
        # regen is True when *all* required channels were re-synced
        # set it tue true after a grace period of 4 hours
        if not regen and oneNewerTimestamp > 0 and time.time() - oneNewerTimestamp > 4 * 60 * 60:
            log("latest channel sync at: {0}. Grace period over. Regenarate bootstrap repo.".format(oneNewerTimestamp), 2)
            regen = True
        if regen:
            regenerate.append(label)
    return regenerate


def generate_all(options, mgr_bootstrap_data, additional=[]):
    repos = {}
    errors = 0

    log("Generating bootstrap repos for all available products which had changes.")

    repos = generate_repo_view(mgr_bootstrap_data)

    regenerated = 0
    for dest, dists in repos.items():
       labels =  find_dists_for_regeneration(dest, dists, doall=options.flush)
       if options.flush and len(labels) > 0:
           destdir = os.path.normpath(dest)
           if os.path.exists(destdir):
               dirprefix, lastdir = os.path.split(destdir)
               destdirold = os.path.join(dirprefix, "{0}.{1}".format(lastdir, "old"))
               log("FLUSH: move destdir '{0}' to old".format(destdir))
               if not options.dryrun:
                   os.rename(destdir, destdirold)
           doall = True
       for label in labels:
           errors += create_repo(label, options, mgr_bootstrap_data, additional=additional)
           regenerated += 1

    if not regenerated:
        log("Nothing to do.")
    return errors



#################################################################################
### main
#################################################################################

def main():
    # quick check to see if you are a super-user.
    if os.getuid() != 0:
        sys.stderr.write('ERROR: must be root to execute.\n')
        sys.exit(1)

    global LOCK
    try:
        LOCK = rhnLockfile.Lockfile('/run/mgr-create-bootstrap-repo.pid')
    except rhnLockfile.LockfileLockedException:
        sys.stderr.write("ERROR: attempting to run more than one instance of "
                         "mgr-create-bootstrap-repo Exiting.\n")
        sys.exit(1)

    global BETA

    opts, args, mgr_bootstrap_data = cli()
    r = 0

    if opts.auto:
        r = generate_all(opts, mgr_bootstrap_data, additional=args)
    elif opts.interactive:
        label_map = list_labels(mgr_bootstrap_data, force=opts.force)
        if not label_map:
            log("No products available")
            sys.exit(0)

        elabel = None
        while True:
            try:
                elabel = label_map.get(int(input("Enter a number of a product label: ")), '')
                break
            except Exception:
                print("Please enter a number.")

        if elabel not in mgr_bootstrap_data.DATA:
            log_error("'%s' not found" % elabel)
            sys.exit(1)

        if opts.flush:
            destdir = os.path.normpath(mgr_bootstrap_data.DATA[elabel]['DEST'])
            if os.path.exists(destdir):
                dirprefix, lastdir = os.path.split(destdir)
                destdirold = os.path.join(dirprefix, "{0}.{1}".format(lastdir, "old"))
                log("FLUSH: move destdir '{0}' to old".format(destdir))
                if not opts.dryrun:
                    os.rename(destdir, destdirold)
        r = create_repo(elabel, opts, mgr_bootstrap_data, additional=args)
    elif opts.list:
        list_labels(mgr_bootstrap_data, force=opts.force)
    elif opts.create:
        if opts.create not in mgr_bootstrap_data.DATA:
            log_error("'%s' not found" % opts.create)
            sys.exit(1)
        if opts.flush:
            destdir = os.path.normpath(mgr_bootstrap_data.DATA[opts.create]['DEST'])
            if os.path.exists(destdir):
                dirprefix, lastdir = os.path.split(destdir)
                destdirold = os.path.join(dirprefix, "{0}.{1}".format(lastdir, "old"))
                log("FLUSH: move destdir '{0}' to old".format(destdir))
                if not opts.dryrun:
                    os.rename(destdir, destdirold)
        r = create_repo(opts.create, opts, mgr_bootstrap_data, additional=args)
    releaseLOCK()
    return r

if __name__ == '__main__':
    try:
        sys.exit(abs(main() or 0))
    except KeyboardInterrupt:
        sys.stderr.write("\nProcess has been interrupted.\n")
        sys.exit(1)
    except SystemExit as e:
        releaseLOCK()
        sys.exit(e.code)
    except Exception as e:
        releaseLOCK()
        raise

