[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20160705-101-g6aa625a

jturney@sourceware.org jturney@sourceware.org
Sun Oct 8 13:42:00 GMT 2017




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=6aa625a6a83ae615a859f61993696a1665340690

commit 6aa625a6a83ae615a859f61993696a1665340690
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Oct 5 12:23:19 2017 +0100

    Upgrade some checks from a warning to an error
    
    Make "curr: version isn't the most recent non-test: version" and "empty but
    not in _obsolete category" errors.
    
    Fix tests by fixing up package timestamps

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4a10c12553a645bd719c056fdc9b463f1065702f

commit 4a10c12553a645bd719c056fdc9b463f1065702f
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Oct 5 14:25:56 2017 +0100

    Update past_mistakes
    
    Clean out things no longer needed

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=900929448bf1c7852598ebe9ea632faa2a475276

commit 900929448bf1c7852598ebe9ea632faa2a475276
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Oct 5 13:09:40 2017 +0100

    Add strict handling for some more warnings in read_package
    
    In particular, the hyphen-in-version warning is now treated as an error in
    strict (upload) mode.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=52f0a5f18d2173c710eb603ee0675a17f91b9457

commit 52f0a5f18d2173c710eb603ee0675a17f91b9457
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Wed Oct 4 16:07:22 2017 +0100

    Also use irker to notify of important events
    
    Also, don't fail if irkerd isn't running.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3673949b6259839db7526584efbe8de432d41db4

commit 3673949b6259839db7526584efbe8de432d41db4
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Wed Oct 4 15:41:41 2017 +0100

    Add more version comparision tests
    
    Test data taken from RPM


Diff:
---
 TODO                  |    1 -
 calm/calm.py          |    9 ++++-
 calm/irk.py           |   13 ++++--
 calm/package.py       |   26 ++++++++++---
 calm/past_mistakes.py |   47 +----------------------
 test/test_calm.py     |   99 +++++++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 138 insertions(+), 57 deletions(-)

diff --git a/TODO b/TODO
index e82b165..07510d5 100644
--- a/TODO
+++ b/TODO
@@ -1,5 +1,4 @@
 * more than 2 versions possible
-* use irkerd to report when calm failed due to an error?
 * upload a hash at the same time as package, and pass that through to setup.ini
 * mksetupini should have an okmissing option for override.hint which names non-existent versions
 * use ./setup.hint inside the tar file, avoiding all the hint/tar coherence problems
diff --git a/calm/calm.py b/calm/calm.py
index 72325f7..14d4317 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -66,6 +66,7 @@ import time
 from .abeyance_handler import AbeyanceHandler
 from .buffering_smtp_handler import BufferingSMTPHandler
 from . import common_constants
+from . import irk
 from . import maintainers
 from . import package
 from . import pkg2html
@@ -246,7 +247,9 @@ def process_uploads(args, state):
                 for arch in common_constants.ARCHES:
                     # use merged package list
                     state.packages[arch] = merged_packages[arch]
-                    logging.debug("added %d (%s) + %d (noarch) + %d (src) packages from maintainer %s" % (len(scan_result[arch].packages), arch, len(scan_result['noarch'].packages), len(scan_result['src'].packages), name))
+                    msg = "added %d (%s) + %d (noarch) + %d (src) packages from maintainer %s" % (len(scan_result[arch].packages), arch, len(scan_result['noarch'].packages), len(scan_result['src'].packages), name)
+                    logging.debug(msg)
+                    irk.irk(msg)
 
         # record updated reminder times for maintainers
         maintainers.Maintainer.update_reminder_times(mlist)
@@ -488,6 +491,7 @@ def do_daemon(args, state):
     with context:
         logging_setup(args)
         logging.info("calm daemon started, pid %d" % (os.getpid()))
+        irk.irk("calm daemon started")
 
         state.packages = {}
 
@@ -531,6 +535,9 @@ def do_daemon(args, state):
         except Exception as e:
             with mail_logs(args.email, toaddrs=args.email, subject='calm stopping due to unhandled exception', thresholdLevel=logging.ERROR) as leads_email:
                 logging.error("exception %s" % (type(e).__name__), exc_info=True)
+            irk.irk("calm daemon stopped due to unhandled exception")
+        else:
+            irk.irk("calm daemon stopped")
 
         logging.info("calm daemon stopped")
 
diff --git a/calm/irk.py b/calm/irk.py
index b9f8957..47cb112 100755
--- a/calm/irk.py
+++ b/calm/irk.py
@@ -26,13 +26,16 @@ def send(s, target, message):
 
 
 def irk(message, target=DEFAULT_TARGET, server=DEFAULT_SERVER):
-    s = connect(server)
-    if "irc:" not in target and "ircs:" not in target:
-        target = "irc://chat.freenode.net/{0}".format(target)
+    try:
+        s = connect(server)
+        if "irc:" not in target and "ircs:" not in target:
+            target = "irc://chat.freenode.net/{0}".format(target)
 
-    send(s, target, message)
+        send(s, target, message)
 
-    s.close()
+        s.close()
+    except ConnectionRefusedError:
+        pass
 
 
 def main():
diff --git a/calm/package.py b/calm/package.py
index cf9d242..ed4cf00 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -259,11 +259,16 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
                 # we already know P to split unambiguously), but this is a bad
                 # idea.
                 if '-' in v:
-                    lvl = logging.WARNING if p not in past_mistakes.hyphen_in_version else logging.INFO
+                    if p not in past_mistakes.hyphen_in_version:
+                        lvl = strict_lvl
+                        warnings = True
+                    else:
+                        lvl = logging.INFO
                     logging.log(lvl, "file '%s' in package '%s' contains '-' in version" % (f, p))
 
                 if not v[0].isdigit():
-                    logging.warning("file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p))
+                    logging.log(strict_lvl, "file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p))
+                    warnings = True
 
                 # if not there already, add to version-release list
                 vr_list.add('%s-%s' % (v, r))
@@ -329,7 +334,8 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
         packages[p].skip = any(['skip' in version_hints[vr] for vr in vr_list])
 
     elif (len(files) > 0) and (relpath.count(os.path.sep) > 1):
-        logging.warning("no .hint files in %s but has files: %s" % (dirpath, ', '.join(files)))
+        logging.log(strict_lvl, "no .hint files in %s but has files: %s" % (dirpath, ', '.join(files)))
+        warnings = True
 
     if strict:
         return warnings
@@ -553,13 +559,17 @@ def validate_packages(args, packages):
         elif 'curr' not in packages[p].stability and 'curr' not in getattr(args, 'okmissing', []):
             logging.warning("package '%s' doesn't have a curr version" % (p))
 
-        # warn if the curr: version isn't the most recent non-test: version
+        # error if the curr: version isn't the most recent non-test: version
         for v in sorted(packages[p].vermap.keys(), key=lambda v: packages[p].vermap[v]['mtime'], reverse=True):
             if 'test' in packages[p].version_hints[v]:
                 continue
 
             if packages[p].stability['curr'] != v:
-                lvl = logging.WARNING if p not in past_mistakes.mtime_anomalies else logging.DEBUG
+                if p in past_mistakes.mtime_anomalies:
+                    lvl = logging.DEBUG
+                else:
+                    lvl = logging.ERROR
+                    error = True
                 logging.log(lvl, "package '%s' version '%s' is most recent non-test version, but version '%s' is curr:" % (p, v, packages[p].stability['curr']))
 
             break
@@ -584,7 +594,11 @@ def validate_packages(args, packages):
                     if 'source' not in packages[p].vermap[vr]:
                         if 'install' in packages[p].vermap[vr]:
                             if packages[p].tars[packages[p].vermap[vr]['install']].is_empty:
-                                lvl = logging.WARNING if p not in past_mistakes.empty_but_not_obsolete else logging.DEBUG
+                                if p in past_mistakes.empty_but_not_obsolete:
+                                    lvl = logging.DEBUG
+                                else:
+                                    lvl = logging.ERROR
+                                    error = True
                                 logging.log(lvl, "package '%s' version '%s' has empty install tar file and no source, but it's not in the _obsolete category" % (p, vr))
 
     # make another pass to verify a source tarfile exists for every install
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index ab925a1..de99439 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -114,19 +114,15 @@ nonunique_versions = [
     'gnome-panel-doc',
     'gtk2.0-engines-svg',
     'guile-doc',
-    'guile-gv',                   # dropped pending guile-2
-    'info',                       # something went wrong with package build?
     'kdepasswd',                  # dropped from split kde-baseapps
     'kdialog',                    # split out from kde-baseapps
     'keditbookmarks',             # split out from kde-baseapps
     'kexi',                       # split out from calligra
     'kfind',                      # split out from kde-baseapps
     'kfilereplace',               # split out from kdewebdev
-    'khelpcenter',                # split out from kde-runtime
     'kimagemapeditor',            # split out from kdewebdev
     'klinkstatus',                # split out from kdewebdev
     'konqueror',                  # split out from kde-baseapps
-    'konqueror-devel',            # split out from kde-baseapps
     'libatomic_ops-devel',        # separated out from libgc
     'libcaca-doc',                # dropped pending fix for current doxygen
     'libfltk-doc',
@@ -139,68 +135,31 @@ nonunique_versions = [
     'minizip',
     'mutter-doc',
     'ocaml-camlp4',               # ocaml-camlp4 removed from ocaml distribution after 4.01.0
-    'ocaml-gv',                   # dropped pending ocaml cleanup
     'okular4-part',               # changed to okular5-part in 17.04
-    'python-clang',               # split out from clang
-    'python3-clang',              # split out from clang
     'python-spiceclientgtk',      # gtk2 dropped from spice-gtk
     'sng-debuginfo',
     'socat-debuginfo',            # debuginfo for test version when curr has no debuginfo
     'sqlite3-zlib',               # sqlite3-zlib removed in 3.8.10, use sqlite3-compress instead
-    'texinfo-debuginfo',          # something went wrong with package build?
     'w3m-img',
 ]
 
 # packages with an empty install file, no source, but aren't obsolete
 empty_but_not_obsolete = [
-    'gcc-java',        # gcc-java will be obsolete in test version 6.3
-    'gdal-debuginfo',  # version 2.1.2-1 has empty debuginfo
-    'libgcj-common',
-    'libgcj16',
-    'libpopt0',        # version 1.16-1 was empty (x8_64)
-    'libpopt-devel',   #
-    'mutt-debuginfo',  # version 1.7.2-1 has empty debuginfo
+    'libpopt0',        # version 1.16-1 was empty
+    'libpopt-devel',   # version 1.16-1 was empty (x86_64)
 ]
 
 # packages with timestamp anomalies
 mtime_anomalies = [
-    'gcc-ada',
-    'gcc-cilkplus',
-    'gcc-core',
-    'gcc-debuginfo',
-    'gcc-fortran',
-    'gcc-g++',
     'gcc-java',
-    'gcc-objc',
-    'gcc-objc++',
     'gcc-tools-epoch2-autoconf',
     'glproto',
     'gv-debuginfo',
-    'info',
-    'libatomic1',
-    'libcilkrts5',
-    'libgcc1',
     'libgcj-common',
     'libgcj16',
-    'libgfortran3',
-    'libgomp1',
-    'libobjc4',
-    'libquadmath0',
-    'libssp0',
-    'libstdc++6',
-    'opencv',  # 2.4.11-1 was created when changed from source-only to install
     'python-gtk2.0',
     'python-gtk2.0-demo',
     'python-gtk2.0-devel',
-    'subversion',
-    'subversion-debuginfo',
-    'subversion-devel',
-    'subversion-gnome',
-    'subversion-httpd',
-    'subversion-perl',
-    'subversion-python',
-    'subversion-ruby',
-    'subversion-tools',
-    'texinfo-debuginfo',
+    'python-wx',  # timestamps reset when split out from wxWidgets
     'xextproto',
 ]
diff --git a/test/test_calm.py b/test/test_calm.py
index d1cf3af..0f30e85 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -172,6 +172,68 @@ class CalmTest(unittest.TestCase):
             ["2a", "2.0", -1],
             ["1.0", "1.fc4", 1],
             ["3.0.0_fc", "3.0.0.fc", 0],
+            # from RPM tests
+            ["1.0", "1.0", 0],
+            ["1.0", "2.0", -1],
+            ["2.0", "1.0", 1],
+            ["2.0.1", "2.0.1", 0],
+            ["2.0", "2.0.1", -1],
+            ["2.0.1", "2.0", 1],
+            ["2.0.1a", "2.0.1a", 0],
+            ["2.0.1a", "2.0.1", 1],
+            ["2.0.1", "2.0.1a", -1],
+            ["5.5p1", "5.5p1", 0],
+            ["5.5p1", "5.5p2", -1],
+            ["5.5p2", "5.5p1", 1],
+            ["5.5p10", "5.5p10", 0],
+            ["5.5p1", "5.5p10", -1],
+            ["5.5p10", "5.5p1", 1],
+            ["10xyz", "10.1xyz", -1],
+            ["10.1xyz", "10xyz", 1],
+            ["xyz10", "xyz10", 0],
+            ["xyz10", "xyz10.1", -1],
+            ["xyz10.1", "xyz10", 1],
+            ["xyz.4", "xyz.4", 0],
+            ["xyz.4", "8", -1],
+            ["8", "xyz.4", 1],
+            ["xyz.4", "2", -1],
+            ["2", "xyz.4", 1],
+            ["5.5p2", "5.6p1", -1],
+            ["5.6p1", "5.5p2", 1],
+            ["5.6p1", "6.5p1", -1],
+            ["6.5p1", "5.6p1", 1],
+            ["6.0.rc1", "6.0", 1],
+            ["6.0", "6.0.rc1", -1],
+            ["10b2", "10a1", 1],
+            ["10a2", "10b2", -1],
+            ["1.0aa", "1.0aa", 0],
+            ["1.0a", "1.0aa", -1],
+            ["1.0aa", "1.0a", 1],
+            ["10.0001", "10.0001", 0],
+            ["10.0001", "10.1", 0],
+            ["10.1", "10.0001", 0],
+            ["10.0001", "10.0039", -1],
+            ["10.0039", "10.0001", 1],
+            ["4.999.9", "5.0", -1],
+            ["5.0", "4.999.9", 1],
+            ["20101121", "20101121", 0],
+            ["20101121", "20101122", -1],
+            ["20101122", "20101121", 1],
+            ["2_0", "2_0", 0],
+            ["2.0", "2_0", 0],
+            ["2_0", "2.0", 0],
+            ["a", "a", 0],
+            ["a+", "a+", 0],
+            ["a+", "a_", 0],
+            ["a_", "a+", 0],
+            ["+a", "+a", 0],
+            ["+a", "_a", 0],
+            ["_a", "+a", 0],
+            ["+_", "+_", 0],
+            ["_+", "+_", 0],
+            ["_+", "_+", 0],
+            ["+", "_", 0],
+            ["_", "+", 0],
         ]
 
         for d in test_data:
@@ -314,6 +376,43 @@ class CalmTest(unittest.TestCase):
         # remove !ready files
         os.system("find testdata/homes -name !ready -exec rm {} \;")
 
+        # fix up package timestamps so highest version is also latest
+        # (git doesn't store timestamps, so they will all be dated the time of checkout)
+        relarea_x86 = os.path.join('testdata', 'relarea', 'x86', 'release')
+        relarea_noarch = os.path.join('testdata', 'relarea', 'noarch', 'release')
+        touches = [(os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.0-1.tar.xz'), '2016-11-01'),
+                   (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.0-1-src.tar.xz'), '2016-11-01'),
+                   (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.1-1.tar.xz'), '2016-11-02'),
+                   (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.1-1-src.tar.xz'), '2016-11-02'),
+                   (os.path.join(relarea_x86, 'cygwin', 'cygwin-debuginfo', 'cygwin-debuginfo-2.2.0-1.tar.xz'), '2016-11-01'),
+                   (os.path.join(relarea_x86, 'cygwin', 'cygwin-debuginfo', 'cygwin-debuginfo-2.2.1-1.tar.xz'), '2016-11-02'),
+                   (os.path.join(relarea_x86, 'cygwin', 'cygwin-devel', 'cygwin-devel-2.2.0-1.tar.xz'), '2016-11-01'),
+                   (os.path.join(relarea_x86, 'cygwin', 'cygwin-devel', 'cygwin-devel-2.2.1-1.tar.xz'), '2016-11-02'),
+                   (os.path.join(relarea_x86, 'base-cygwin', 'base-cygwin-3.6-1.tar.xz'), '2016-11-02'),
+                   (os.path.join(relarea_x86, 'per-version', 'per-version-4.0-1.tar.xz'), '2017-04-09'),
+                   (os.path.join(relarea_x86, 'per-version', 'per-version-4.0-1-src.tar.xz'), '2017-04-09'),
+                   (os.path.join(relarea_x86, 'rpm-doc', 'rpm-doc-4.1-2.tar.bz2'), '2016-11-02'),
+                   (os.path.join(relarea_x86, 'rpm-doc', 'rpm-doc-4.1-2-src.tar.bz2'), '2016-11-02'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-240-1.tar.xz'), '2017-04-07'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-240-1-src.tar.xz'), '2017-04-07'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-242-0.tar.xz'), '2017-04-08'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-242-0-src.tar.xz'), '2017-04-08'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-243-0.tar.xz'), '2017-04-09'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-243-0-src.tar.xz'), '2017-04-09'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-250-0.tar.xz'), '2017-04-10'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-250-0-src.tar.xz'), '2017-04-10'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-251-0.tar.xz'), '2017-04-09'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-251-0-src.tar.xz'), '2017-04-09'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-260-0.tar.xz'), '2017-04-12'),
+                   (os.path.join(relarea_x86, 'staleversion', 'staleversion-260-0-src.tar.xz'), '2017-04-12'),
+                   (os.path.join(relarea_x86, 'keychain', 'keychain-2.6.8-1.tar.bz2'), '2016-11-02'),
+                   (os.path.join(relarea_x86, 'keychain', 'keychain-2.6.8-1-src.tar.bz2'), '2016-11-02'),
+                   (os.path.join(relarea_noarch, 'perl-Net-SMTP-SSL', 'perl-Net-SMTP-SSL-1.03-1.tar.xz'), '2016-11-01'),
+                   (os.path.join(relarea_noarch, 'perl-Net-SMTP-SSL', 'perl-Net-SMTP-SSL-1.03-1-src.tar.xz'), '2016-11-01'),
+        ]
+        for (f, t) in touches:
+            os.system('touch %s -d %s' % (f, t))
+
 
 if __name__ == '__main__':
     logging.getLogger().setLevel(logging.INFO)



More information about the Cygwin-apps-cvs mailing list