diff options
| -rw-r--r-- | images/CMakeLists.txt | 52 | ||||
| -rw-r--r-- | images/README.md | 83 | ||||
| -rwxr-xr-x | images/create_imgs_package.py | 110 | ||||
| -rwxr-xr-x | images/make_zip.sh | 51 | ||||
| -rwxr-xr-x | images/populate_images.py | 38 | ||||
| -rw-r--r-- | images/uhdimgs.py | 23 |
6 files changed, 96 insertions, 261 deletions
diff --git a/images/CMakeLists.txt b/images/CMakeLists.txt deleted file mode 100644 index 229f0c422..000000000 --- a/images/CMakeLists.txt +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright 2010-2012 Ettus Research LLC -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# - -######################################################################## -# Setup Project -######################################################################## -#OPTION(UHD_PATCH_OVERRIDE "Use this string to force a patch level version string." OFF) -MESSAGE(STATUS ${UHD_PATCH_OVERRIDE}) -IF(DEFINED UHD_PATCH_OVERRIDE) - SET(UHD_VERSION_PATCH_OVERRIDE ${UHD_PATCH_OVERRIDE}) -ENDIF(DEFINED UHD_PATCH_OVERRIDE) -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(UHD-images NONE) -LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../host/cmake/Modules) -INCLUDE(UHDVersion) #sets version variables (used below) -EXECUTE_PROCESS(COMMAND "date" OUTPUT_VARIABLE DATETIME_NOW OUTPUT_STRIP_TRAILING_WHITESPACE) #sets the date - -######################################################################## -# Setup CPack -######################################################################## -SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Ettus Research - USRP Hardware Driver Images") -SET(CPACK_PACKAGE_VENDOR "Ettus Research LLC") -SET(CPACK_PACKAGE_CONTACT "support@ettus.com") -SET(CPACK_PACKAGE_VERSION ${UHD_VERSION}) -SET(CPACK_PACKAGE_FILE_NAME "uhd-images_${UHD_VERSION}") -SET(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "all") -INCLUDE(CPack) #include after setting vars -MESSAGE(STATUS "Version: ${UHD_VERSION}") - -######################################################################## -# Install Images -######################################################################## - -#tag the images with a version number (something identifiable) -FILE(WRITE ${CMAKE_SOURCE_DIR}/images/${UHD_VERSION_MAJOR}.${UHD_VERSION_API}.${UHD_VERSION_ABI}.${UHD_VERSION_PATCH}.tag "${UHD_VERSION}\n${DATETIME_NOW}\n") -INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/images DESTINATION share/uhd) -INSTALL(FILES ${CMAKE_SOURCE_DIR}/../host/LICENSE DESTINATION share/uhd/images) -INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/manifest.txt DESTINATION share/uhd/images) diff --git a/images/README.md b/images/README.md index 6c96fde7b..f9ffeb667 100644 --- a/images/README.md +++ b/images/README.md @@ -1,83 +1,20 @@ UHD Firmware and FPGA Images Builder =============================================================== -The images directory contains the following: -* A Makefile for building firmware and FPGA images -* Scripts to load all the binaries for this current commit, as - well as create a new images package from the images in subdir - images/ +The images directory is an aid to prepare images packages. -This provides a clean and organized way to build all of the firmware and FPGA -images, the source code for which is in the `firmware` and `fpga` directories -one level above this, and also maintains a linkage between images and git commits. +Building the actual FPGA images is not handled with these tools. -Building the binaries ---------------------- +## Making image packages -The Makefile and build systems for the images are *probably* Unix-specific. -It's best to build the images on a Unix system with standard build tools. The -CMake package target will create an installable images package for your system. +At any point in time, when the FPGA images change, you will need to create new +FPGA images packages. Use the `package_images.py` script for that purpose. -__To build the images (unix):__ +## Create image tarballs for releases -1. `make clean` -2. `make images` +If you're on a release tag and want to create images packages for uploading to +github, run -__Fedora note:__ - -The sdcc binaries are prefixed with "sdcc-" which breaks the build. -However, /usr/libexec/sdcc contains properly named sdcc binaries. -`export PATH=${PATH}:/usr/libexec/sdcc` - - -Updating binaries ------------------ - -This goes two ways: - -1. Loading the correct binaries for this commit -2. Updating the binary package on this branch - - -### Loading the correct binaries ### - -If you check out a branch or commit, you might want to use the exact same -binaries that were used when this branch or commit was generated. -To do this, run `populate_images.py`. This will either download the correct -images package from a web server or from a local directory if -`UHD_IMAGES_BASE_URL` is set. - -### Updating the binaries ### - -If you have commited changes to this branch that require new images, you -should probably update those. -Simply copy the new image binaries into images/. If necessary, run -`populate_images.py` before you do any of this, so all the untouched -images are the correct version. Then, run `create_imgs_package.py --commit "COMMIT MSG"` -to create a new ZIP file and commit the info. -If `UHD_IMAGES_BASE_URL` is set and is a local directory, it will move -the ZIP file to this directory after creating it. - -### Updating the binaries ### - -Typical workflow: - -1. Check out a branch or commit (git checkout FOO) -2. Update the images/ subdir (`populate_images.py`) -3. Do some coding, and commit those changes (`git commit`) -4. Copy new binaries to images/ -5. Commit the new binaries: `create_imgs_package.py --commit "Updated images on branch X"` - -### The CPack system ### - -Underlying `create_imgs_package.py` is a CPack system, which can be manually -invoked by: - -1. `mkdir build` -2. `cd build` -3. `cmake -DCPACK_GENERATOR=<type> ../` -4. `make package` - -The package generator types are described here: -http://www.cmake.org/Wiki/CMake:CPackPackageGenerators + $ ./create_imgs_package.py +It'll take a few minutes to download and zip them all up. diff --git a/images/create_imgs_package.py b/images/create_imgs_package.py index 82b7d65d4..50882287b 100755 --- a/images/create_imgs_package.py +++ b/images/create_imgs_package.py @@ -19,86 +19,66 @@ Command-line utility to create a .zip-file with the current image set. """ +from __future__ import print_function import re import os -import uhdimgs -import glob import subprocess import argparse import shutil - -def clear_img_dir(img_root_dir): - """ Removes non-image files from the images dir """ - globs = ["*.tag", "LICENSE"] - for the_glob in globs: - for filename in glob.iglob(os.path.join(img_root_dir, the_glob)): - print 'Removing file from images directory: ', filename - os.unlink(filename) - -def get_zipfilename_from_cpack_output(cpoutput): - """ Parses the output of the ZIP-file creating script - and scrapes the actual file name. """ - regex = re.compile("\/build\/(?P<filename>[^\/]+\.zip)") - results = regex.search(cpoutput) - return results.group('filename') +import uhdimgs def parse_args(): """ Parse args, duh """ - parser = argparse.ArgumentParser(description='Link the current set of images to this commit.') - parser.add_argument('--commit', default=None, - help='Supply a commit message to the changes to host/CMakeLists.txt.') - parser.add_argument('-r', '--release-mode', default="", - help='Specify UHD_RELEASE_MODE. Typically "release" or "rc1" or similar.') - parser.add_argument('--skip-edit', default=False, action='store_true', - help='Do not edit the CMakeLists.txt file.') - parser.add_argument('--skip-move', default=False, action='store_true', - help='Do not move the archives after creating them.') - parser.add_argument('--patch', help='Override patch version number.') + parser = argparse.ArgumentParser() + parser.add_argument( + '--version', + help="Specify version. Will detect from tag otherwise." + ) return parser.parse_args() -def move_zip_to_repo(base_url, zipfilename): - final_destination = os.path.join(base_url, zipfilename) - if os.path.exists(final_destination): - print "WARNING: A file with name {0} is already in the images repository.".format(zipfilename) - print "Overwrite? [y/N]", - ans = raw_input() - if ans.strip().upper() != 'Y': - return - os.unlink(final_destination) - shutil.move(zipfilename, base_url) +def download_images(img_root_dir): + """ + Run the images downloader + """ + import populate_images + populate_images.download_images(img_root_dir) + +def get_version(): + """ + Figure out version based on tag. + """ + try: + git_cmd = ['git', 'describe', '--abbrev=0', '--tags'] + git_output = subprocess.check_output(git_cmd) + except subprocess.CalledProcessError as ex: + print(ex.output) + exit(1) + print("Detected tag: {}".format(git_output)) + version_mobj = re.search(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]', git_output) + if version_mobj is None: + print("Error: Failure to resolve version from tag!") + exit(1) + return version_mobj.group(0) def main(): - " Go, go, go! " + """ Go, go, go! """ args = parse_args() img_root_dir = os.path.join(uhdimgs.get_images_dir(), 'images') - os.chdir(uhdimgs.get_images_dir()) - print "== Clearing out the images directory..." - clear_img_dir(img_root_dir) - print "== Creating archives..." - cpack_cmd = ["./make_zip.sh",] - cpack_cmd.append(args.release_mode) - if args.patch is not None: - cpack_cmd.append("-DUHD_PATCH_OVERRIDE={}".format(args.patch)) + print("== Clearing out the images directory...") + shutil.rmtree(img_root_dir) + print("== Downloading images...") + download_images(img_root_dir) + print("== Determining version...") + version = args.version if args.version is not None else get_version() + print("Version string: {}".format(version)) + print("== Creating archives...") + archive_cmd = ["./make_zip.sh", version] try: - cpack_output = subprocess.check_output(cpack_cmd) - except subprocess.CalledProcessError as e: - print e.output - raise SystemExit, 1 - zipfilename = get_zipfilename_from_cpack_output(cpack_output) - print "Filename: ", zipfilename - print "== Calculating MD5 sum of ZIP archive..." - md5 = uhdimgs.md5_checksum(zipfilename) - print 'MD5: ', md5 - base_url = uhdimgs.get_base_url() - if not args.skip_move and uhdimgs.base_url_is_local(base_url) and os.access(base_url, os.W_OK): - print "== Moving ZIP file to {0}...".format(base_url) - move_zip_to_repo(base_url, zipfilename) - print "== Updating CMakeLists.txt..." - uhdimgs.update_main_cmake_file(md5, zipfilename) - if args.commit is not None: - print "== Committing changes..." - subprocess.check_call(['git', 'commit', '-m', args.commit, uhdimgs.get_cmake_main_file()]) - print "== Done!" + subprocess.call(archive_cmd) + except subprocess.CalledProcessError as ex: + print(ex.output) + exit(1) + print("== Done!") if __name__ == "__main__": main() diff --git a/images/make_zip.sh b/images/make_zip.sh index e9ef802f7..40ba61df5 100755 --- a/images/make_zip.sh +++ b/images/make_zip.sh @@ -11,46 +11,31 @@ if [ ! -e 'images' ]; then fi # Remove cruft before proceeding: -if [ -e 'build' ]; then - echo 'Please remove build subdirectory before proceeding.' - exit 1 -fi -if [ -e "images/LICENSE" ]; then - rm images/LICENSE -fi TAGFILES=`ls images/*.tag 2>/dev/null` if [ -n "$TAGFILES" ]; then rm $TAGFILES fi -# Enter build dir -mkdir build -cd build +# Copy LICENSE file +cp ../host/LICENSE images/ -# Run the CPack process (ZIP file) -cmake .. -DCPACK_GENERATOR=ZIP -DUHD_RELEASE_MODE="$1" $2 .. -make package -mv uhd-images*.zip .. +ARCHIVE_SUFFIX=$1 +ARCHIVE_NAME="uhd-images_${ARCHIVE_SUFFIX}" +echo "Creating images archive: ${ARCHIVE_NAME}" -# Run the CPack process (tarball) -cmake .. -DCPACK_GENERATOR=TGZ -DUHD_RELEASE_MODE="$1" $2 .. -make package -mv uhd-images*.tar.gz .. +cp -r images $ARCHIVE_NAME -# Move images to here and clean up after us: -cd .. -rm -r build -rm images/*.tag +# Now zip 'em up: +echo "Creating ZIP archive..." +zip -r $ARCHIVE_NAME.zip $ARCHIVE_NAME +echo "Creating tar.gz archive..." +tar zcvf $ARCHIVE_NAME.tar.gz $ARCHIVE_NAME +echo "Creating tar.xz archive..." +tar Jcvf $ARCHIVE_NAME.tar.xz $ARCHIVE_NAME +ALL_ARCHIVES=`ls $ARCHIVE_NAME.*` +sha256sum $ALL_ARCHIVES > $ARCHIVE_NAME.sha256 +md5sum $ALL_ARCHIVES > $ARCHIVE_NAME.md5 -TGZ_ARCHIVE_NAME=`ls *.tar.gz | tail -n1` - -# CMake can't do xz, so do it by hand if possible -XZ_EXECUTABLE=`which xz` -if [ $? -eq 0 ]; then - XZ_ARCHIVE_NAME=`echo $TGZ_ARCHIVE_NAME | sed "s/gz\>/xz/"` - echo "Writing .xz tarball to $XZ_ARCHIVE_NAME ..." - gunzip --to-stdout $TGZ_ARCHIVE_NAME | xz - > $XZ_ARCHIVE_NAME -fi +# Clean up archive directory +rm -r $ARCHIVE_NAME -MD5_FILE_NAME=`echo $TGZ_ARCHIVE_NAME | sed "s/tar.gz\>/md5/"` -md5sum uhd-images* > $MD5_FILE_NAME diff --git a/images/populate_images.py b/images/populate_images.py index 20fcb7028..adf75ec8b 100755 --- a/images/populate_images.py +++ b/images/populate_images.py @@ -1,47 +1,41 @@ #!/usr/bin/env python # # Copyright 2014 Ettus Research LLC +# Copyright 2018 Ettus Research, a National Instruments Company # -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. +# SPDX-License-Identifier: GPL-3.0-or-later # """ Populates the current directory with a valid set of binaries for the current commit. """ +from __future__ import print_function import os import subprocess import uhdimgs -def main(): +def download_images(img_root_dir=None): " Go, go, go! " # Switch to correct dir - img_root_dir = os.path.join(uhdimgs.get_images_dir(), 'images') + img_root_dir = img_root_dir or os.path.join(uhdimgs.get_images_dir(), 'images') + if not os.path.isdir(img_root_dir): + print("== Creating images directory...") + os.mkdir(img_root_dir) os.chdir(uhdimgs.get_images_dir()) - print "== Starting download..." + print("== Starting download...") try: downloader_cmd = [ - 'python', - '../host/utils/uhd_images_downloader.py.in', - '-i', img_root_dir, - '-m', 'manifest.txt' + 'python', + '../host/utils/uhd_images_downloader.py.in', + '-i', img_root_dir, + '-m', 'manifest.txt' ] subprocess.check_call(downloader_cmd) except (subprocess.CalledProcessError, OSError): - print "[ERROR] Failed to run downloader script." + print("[ERROR] Failed to run downloader script.") exit(1) - print "== Done!" + print("== Done!") if __name__ == "__main__": - main() + download_images() diff --git a/images/uhdimgs.py b/images/uhdimgs.py index 77da36c0d..fa0b0ac86 100644 --- a/images/uhdimgs.py +++ b/images/uhdimgs.py @@ -1,24 +1,15 @@ #!/usr/bin/env python # # Copyright 2014 Ettus Research LLC +# Copyright 2018 Ettus Research, a National Instruments Company # -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. +# SPDX-License-Identifier: GPL-3.0-or-later # """ Utility module for packaging and handling UHD binary images. """ +from __future__ import print_function import re import os import sys @@ -75,12 +66,12 @@ def get_total_md5(img_dir): for filename in filenames: md5_obj.update(open(os.path.join(root, filename), 'rb').read()) sys.stdout.write('.') - sys.stdout.flus() + sys.stdout.flush() for dirname in dirnames: _update_md5_for_dir_recursive(os.path.join(root, dirname), md5_obj) md5 = hashlib.md5() _update_md5_for_dir_recursive(img_dir, md5) - print "" + print("") return md5.hexdigest() def md5_checksum(filePath): @@ -94,6 +85,6 @@ def md5_checksum(filePath): break m.update(data) return m.hexdigest() - except Exception, e: - print "Failed to calculated MD5 sum of: %s (%s)" % (filePath, e) + except Exception as e: + print("Failed to calculated MD5 sum of: %s (%s)" % (filePath, e)) raise e |