images: Update scripts for new release model

- populate_images.py: Made Py3k safe
- uhdimgs.py: Make Py3k-safe
- create_imgs_package.py: Rewrote to build release images packages
- make_zip.sh: Skip CMake, just zip up the files
- Updated README to reflect all the changes

6 files changed, 96 insertions, 261 deletions

diff --git a/images/CMakeLists.txt b/images/CMakeLists.txt
deleted file mode 100644
index 229f0c422..000000000
--- a/images/CMakeLists.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Copyright 2010-2012 Ettus Research LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-########################################################################
-# Setup Project
-########################################################################
-#OPTION(UHD_PATCH_OVERRIDE "Use this string to force a patch level version string." OFF)
-MESSAGE(STATUS ${UHD_PATCH_OVERRIDE})
-IF(DEFINED UHD_PATCH_OVERRIDE)
-    SET(UHD_VERSION_PATCH_OVERRIDE ${UHD_PATCH_OVERRIDE})
-ENDIF(DEFINED UHD_PATCH_OVERRIDE)
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
-PROJECT(UHD-images NONE)
-LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../host/cmake/Modules)
-INCLUDE(UHDVersion) #sets version variables (used below)
-EXECUTE_PROCESS(COMMAND "date" OUTPUT_VARIABLE DATETIME_NOW OUTPUT_STRIP_TRAILING_WHITESPACE) #sets the date
-
-########################################################################
-# Setup CPack
-########################################################################
-SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Ettus Research - USRP Hardware Driver Images")
-SET(CPACK_PACKAGE_VENDOR              "Ettus Research LLC")
-SET(CPACK_PACKAGE_CONTACT             "support@ettus.com")
-SET(CPACK_PACKAGE_VERSION ${UHD_VERSION})
-SET(CPACK_PACKAGE_FILE_NAME "uhd-images_${UHD_VERSION}")
-SET(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "all")
-INCLUDE(CPack) #include after setting vars
-MESSAGE(STATUS "Version: ${UHD_VERSION}")
-
-########################################################################
-# Install Images
-########################################################################
-
-#tag the images with a version number (something identifiable)
-FILE(WRITE ${CMAKE_SOURCE_DIR}/images/${UHD_VERSION_MAJOR}.${UHD_VERSION_API}.${UHD_VERSION_ABI}.${UHD_VERSION_PATCH}.tag "${UHD_VERSION}\n${DATETIME_NOW}\n")
-INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/images DESTINATION share/uhd)
-INSTALL(FILES ${CMAKE_SOURCE_DIR}/../host/LICENSE DESTINATION share/uhd/images)
-INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/manifest.txt DESTINATION share/uhd/images)
diff --git a/images/README.md b/images/README.md
index 6c96fde7b..f9ffeb667 100644
--- a/images/README.md
+++ b/images/README.md
@@ -1,83 +1,20 @@
 UHD Firmware and FPGA Images Builder
 ===============================================================
 
-The images directory contains the following:
-* A Makefile for building firmware and FPGA images
-* Scripts to load all the binaries for this current commit, as
-  well as create a new images package from the images in subdir
-  images/
+The images directory is an aid to prepare images packages.
 
-This provides a clean and organized way to build all of the firmware and FPGA
-images, the source code for which is in the `firmware` and `fpga` directories
-one level above this, and also maintains a linkage between images and git commits.
+Building the actual FPGA images is not handled with these tools.
 
-Building the binaries
----------------------
+## Making image packages
 
-The Makefile and build systems for the images are *probably* Unix-specific.
-It's best to build the images on a Unix system with standard build tools.  The
-CMake package target will create an installable images package for your system.
+At any point in time, when the FPGA images change, you will need to create new
+FPGA images packages. Use the `package_images.py` script for that purpose.
 
-__To build the images (unix):__
+## Create image tarballs for releases
 
-1. `make clean`
-2. `make images`
+If you're on a release tag and want to create images packages for uploading to
+github, run
 
-__Fedora note:__
-
-The sdcc binaries are prefixed with "sdcc-" which breaks the build.
-However, /usr/libexec/sdcc contains properly named sdcc binaries.
-`export PATH=${PATH}:/usr/libexec/sdcc`
-
-
-Updating binaries
------------------
-
-This goes two ways:
-
-1. Loading the correct binaries for this commit
-2. Updating the binary package on this branch
-
-
-### Loading the correct binaries ###
-
-If you check out a branch or commit, you might want to use the exact same
-binaries that were used when this branch or commit was generated.
-To do this, run `populate_images.py`. This will either download the correct
-images package from a web server or from a local directory if
-`UHD_IMAGES_BASE_URL` is set.
-
-### Updating the binaries ###
-
-If you have commited changes to this branch that require new images, you
-should probably update those.
-Simply copy the new image binaries into images/. If necessary, run
-`populate_images.py` before you do any of this, so all the untouched
-images are the correct version. Then, run `create_imgs_package.py --commit "COMMIT MSG"`
-to create a new ZIP file and commit the info.
-If `UHD_IMAGES_BASE_URL` is set and is a local directory, it will move
-the ZIP file to this directory after creating it.
-
-### Updating the binaries ###
-
-Typical workflow:
-
-1. Check out a branch or commit (git checkout FOO)
-2. Update the images/ subdir (`populate_images.py`)
-3. Do some coding, and commit those changes (`git commit`)
-4. Copy new binaries to images/
-5. Commit the new binaries: `create_imgs_package.py --commit "Updated images on branch X"`
-
-### The CPack system ###
-
-Underlying `create_imgs_package.py` is a CPack system, which can be manually
-invoked by:
-
-1. `mkdir build`
-2. `cd build`
-3. `cmake -DCPACK_GENERATOR=<type> ../`
-4. `make package`
-
-The package generator types are described here:
-http://www.cmake.org/Wiki/CMake:CPackPackageGenerators
+    $ ./create_imgs_package.py
 
+It'll take a few minutes to download and zip them all up.
diff --git a/images/create_imgs_package.py b/images/create_imgs_package.py
index 82b7d65d4..50882287b 100755
--- a/images/create_imgs_package.py
+++ b/images/create_imgs_package.py
@@ -19,86 +19,66 @@
 Command-line utility to create a .zip-file with the current image set.
 """
 
+from __future__ import print_function
 import re
 import os
-import uhdimgs
-import glob
 import subprocess
 import argparse
 import shutil
-
-def clear_img_dir(img_root_dir):
-    """ Removes non-image files from the images dir """
-    globs = ["*.tag", "LICENSE"]
-    for the_glob in globs:
-        for filename in glob.iglob(os.path.join(img_root_dir, the_glob)):
-            print 'Removing file from images directory: ', filename
-            os.unlink(filename)
-
-def get_zipfilename_from_cpack_output(cpoutput):
-    """ Parses the output of the ZIP-file creating script
-    and scrapes the actual file name. """
-    regex = re.compile("\/build\/(?P<filename>[^\/]+\.zip)")
-    results = regex.search(cpoutput)
-    return results.group('filename')
+import uhdimgs
 
 def parse_args():
     """ Parse args, duh """
-    parser = argparse.ArgumentParser(description='Link the current set of images to this commit.')
-    parser.add_argument('--commit', default=None,
-                       help='Supply a commit message to the changes to host/CMakeLists.txt.')
-    parser.add_argument('-r', '--release-mode', default="",
-                       help='Specify UHD_RELEASE_MODE. Typically "release" or "rc1" or similar.')
-    parser.add_argument('--skip-edit', default=False, action='store_true',
-                       help='Do not edit the CMakeLists.txt file.')
-    parser.add_argument('--skip-move', default=False, action='store_true',
-                       help='Do not move the archives after creating them.')
-    parser.add_argument('--patch', help='Override patch version number.')
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--version',
+        help="Specify version. Will detect from tag otherwise."
+    )
     return parser.parse_args()
 
-def move_zip_to_repo(base_url, zipfilename):
-    final_destination = os.path.join(base_url, zipfilename)
-    if os.path.exists(final_destination):
-        print "WARNING: A file with name {0} is already in the images repository.".format(zipfilename)
-        print "Overwrite? [y/N]",
-        ans = raw_input()
-        if ans.strip().upper() != 'Y':
-            return
-        os.unlink(final_destination)
-    shutil.move(zipfilename, base_url)
+def download_images(img_root_dir):
+    """
+    Run the images downloader
+    """
+    import populate_images
+    populate_images.download_images(img_root_dir)
+
+def get_version():
+    """
+    Figure out version based on tag.
+    """
+    try:
+        git_cmd = ['git', 'describe', '--abbrev=0', '--tags']
+        git_output = subprocess.check_output(git_cmd)
+    except subprocess.CalledProcessError as ex:
+        print(ex.output)
+        exit(1)
+    print("Detected tag: {}".format(git_output))
+    version_mobj = re.search(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]', git_output)
+    if version_mobj is None:
+        print("Error: Failure to resolve version from tag!")
+        exit(1)
+    return version_mobj.group(0)
 
 def main():
-    " Go, go, go! "
+    """ Go, go, go! """
     args = parse_args()
     img_root_dir = os.path.join(uhdimgs.get_images_dir(), 'images')
-    os.chdir(uhdimgs.get_images_dir())
-    print "== Clearing out the images directory..."
-    clear_img_dir(img_root_dir)
-    print "== Creating archives..."
-    cpack_cmd = ["./make_zip.sh",]
-    cpack_cmd.append(args.release_mode)
-    if args.patch is not None:
-        cpack_cmd.append("-DUHD_PATCH_OVERRIDE={}".format(args.patch))
+    print("== Clearing out the images directory...")
+    shutil.rmtree(img_root_dir)
+    print("== Downloading images...")
+    download_images(img_root_dir)
+    print("== Determining version...")
+    version = args.version if args.version is not None else get_version()
+    print("Version string: {}".format(version))
+    print("== Creating archives...")
+    archive_cmd = ["./make_zip.sh", version]
     try:
-        cpack_output = subprocess.check_output(cpack_cmd)
-    except subprocess.CalledProcessError as e:
-        print e.output
-        raise SystemExit, 1
-    zipfilename = get_zipfilename_from_cpack_output(cpack_output)
-    print "Filename: ", zipfilename
-    print "== Calculating MD5 sum of ZIP archive..."
-    md5 = uhdimgs.md5_checksum(zipfilename)
-    print 'MD5: ', md5
-    base_url = uhdimgs.get_base_url()
-    if not args.skip_move and uhdimgs.base_url_is_local(base_url) and os.access(base_url, os.W_OK):
-        print "== Moving ZIP file to {0}...".format(base_url)
-        move_zip_to_repo(base_url, zipfilename)
-    print "== Updating CMakeLists.txt..."
-    uhdimgs.update_main_cmake_file(md5, zipfilename)
-    if args.commit is not None:
-        print "== Committing changes..."
-        subprocess.check_call(['git', 'commit', '-m', args.commit, uhdimgs.get_cmake_main_file()])
-    print "== Done!"
+        subprocess.call(archive_cmd)
+    except subprocess.CalledProcessError as ex:
+        print(ex.output)
+        exit(1)
+    print("== Done!")
 
 if __name__ == "__main__":
     main()
diff --git a/images/make_zip.sh b/images/make_zip.sh
index e9ef802f7..40ba61df5 100755
--- a/images/make_zip.sh
+++ b/images/make_zip.sh
@@ -11,46 +11,31 @@ if [ ! -e 'images' ]; then
 fi
 
 # Remove cruft before proceeding:
-if [ -e 'build' ]; then
-    echo 'Please remove build subdirectory before proceeding.'
-    exit 1
-fi
-if [ -e "images/LICENSE" ]; then
-	rm images/LICENSE
-fi
 TAGFILES=`ls images/*.tag 2>/dev/null`
 if [ -n "$TAGFILES" ]; then
 	rm $TAGFILES
 fi
 
-# Enter build dir
-mkdir build
-cd build
+# Copy LICENSE file
+cp ../host/LICENSE images/
 
-# Run the CPack process (ZIP file)
-cmake .. -DCPACK_GENERATOR=ZIP -DUHD_RELEASE_MODE="$1" $2 ..
-make package
-mv uhd-images*.zip ..
+ARCHIVE_SUFFIX=$1
+ARCHIVE_NAME="uhd-images_${ARCHIVE_SUFFIX}"
+echo "Creating images archive: ${ARCHIVE_NAME}"
 
-# Run the CPack process (tarball)
-cmake .. -DCPACK_GENERATOR=TGZ -DUHD_RELEASE_MODE="$1" $2 ..
-make package
-mv uhd-images*.tar.gz ..
+cp -r images $ARCHIVE_NAME
 
-# Move images to here and clean up after us:
-cd ..
-rm -r build
-rm images/*.tag
+# Now zip 'em up:
+echo "Creating ZIP archive..."
+zip -r $ARCHIVE_NAME.zip $ARCHIVE_NAME
+echo "Creating tar.gz archive..."
+tar zcvf $ARCHIVE_NAME.tar.gz $ARCHIVE_NAME
+echo "Creating tar.xz archive..."
+tar Jcvf $ARCHIVE_NAME.tar.xz $ARCHIVE_NAME
+ALL_ARCHIVES=`ls $ARCHIVE_NAME.*`
+sha256sum $ALL_ARCHIVES > $ARCHIVE_NAME.sha256
+md5sum $ALL_ARCHIVES > $ARCHIVE_NAME.md5
 
-TGZ_ARCHIVE_NAME=`ls *.tar.gz | tail -n1`
-
-# CMake can't do xz, so do it by hand if possible
-XZ_EXECUTABLE=`which xz`
-if [ $? -eq 0 ]; then
-	XZ_ARCHIVE_NAME=`echo $TGZ_ARCHIVE_NAME | sed "s/gz\>/xz/"`
-	echo "Writing .xz tarball to $XZ_ARCHIVE_NAME ..."
-	gunzip --to-stdout $TGZ_ARCHIVE_NAME | xz - > $XZ_ARCHIVE_NAME
-fi
+# Clean up archive directory
+rm -r $ARCHIVE_NAME
 
-MD5_FILE_NAME=`echo $TGZ_ARCHIVE_NAME | sed "s/tar.gz\>/md5/"`
-md5sum uhd-images* > $MD5_FILE_NAME
diff --git a/images/populate_images.py b/images/populate_images.py
index 20fcb7028..adf75ec8b 100755
--- a/images/populate_images.py
+++ b/images/populate_images.py
@@ -1,47 +1,41 @@
 #!/usr/bin/env python
 #
 # Copyright 2014 Ettus Research LLC
+# Copyright 2018 Ettus Research, a National Instruments Company
 #
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# SPDX-License-Identifier: GPL-3.0-or-later
 #
 """
 Populates the current directory with a valid set of binaries for the
 current commit.
 """
 
+from __future__ import print_function
 import os
 import subprocess
 import uhdimgs
 
-def main():
+def download_images(img_root_dir=None):
     " Go, go, go! "
     # Switch to correct dir
-    img_root_dir = os.path.join(uhdimgs.get_images_dir(), 'images')
+    img_root_dir = img_root_dir or os.path.join(uhdimgs.get_images_dir(), 'images')
+    if not os.path.isdir(img_root_dir):
+        print("== Creating images directory...")
+        os.mkdir(img_root_dir)
     os.chdir(uhdimgs.get_images_dir())
-    print "== Starting download..."
+    print("== Starting download...")
     try:
         downloader_cmd = [
-                'python',
-                '../host/utils/uhd_images_downloader.py.in',
-                '-i', img_root_dir,
-                '-m', 'manifest.txt'
+            'python',
+            '../host/utils/uhd_images_downloader.py.in',
+            '-i', img_root_dir,
+            '-m', 'manifest.txt'
         ]
         subprocess.check_call(downloader_cmd)
     except (subprocess.CalledProcessError, OSError):
-        print "[ERROR] Failed to run downloader script."
+        print("[ERROR] Failed to run downloader script.")
         exit(1)
-    print "== Done!"
+    print("== Done!")
 
 if __name__ == "__main__":
-    main()
+    download_images()
diff --git a/images/uhdimgs.py b/images/uhdimgs.py
index 77da36c0d..fa0b0ac86 100644
--- a/images/uhdimgs.py
+++ b/images/uhdimgs.py
@@ -1,24 +1,15 @@
 #!/usr/bin/env python
 #
 # Copyright 2014 Ettus Research LLC
+# Copyright 2018 Ettus Research, a National Instruments Company
 #
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# SPDX-License-Identifier: GPL-3.0-or-later
 #
 """
 Utility module for packaging and handling UHD binary images.
 """
 
+from __future__ import print_function
 import re
 import os
 import sys
@@ -75,12 +66,12 @@ def get_total_md5(img_dir):
             for filename in filenames:
                 md5_obj.update(open(os.path.join(root, filename), 'rb').read())
                 sys.stdout.write('.')
-                sys.stdout.flus()
+                sys.stdout.flush()
             for dirname in dirnames:
                 _update_md5_for_dir_recursive(os.path.join(root, dirname), md5_obj)
     md5 = hashlib.md5()
     _update_md5_for_dir_recursive(img_dir, md5)
-    print ""
+    print("")
     return md5.hexdigest()
 
 def md5_checksum(filePath):
@@ -94,6 +85,6 @@ def md5_checksum(filePath):
                     break
                 m.update(data)
             return m.hexdigest()
-    except Exception, e:
-        print "Failed to calculated MD5 sum of: %s (%s)" % (filePath, e)
+    except Exception as e:
+        print("Failed to calculated MD5 sum of: %s (%s)" % (filePath, e))
         raise e