relase: make github operations perfect

Author: cirosantilli

README.adoc

@@ -293,7 +293,7 @@ If you really want to develop semiconductors, your only choice is to join an uni
 
 ==== About the QEMU Buildroot setup
 
-link:https://en.wikipedia.org/wiki/Buildroot[Buildroot] is a set of `make` scripts that downloads from source and compiles compatible versions of:
+link:https://en.wikipedia.org/wiki/Buildroot[Buildroot] is a set of Make scripts that download and compile from source compatible versions of:
 
 * GCC
 * Linux kernel
@@ -553,7 +553,7 @@ Some times it works with the host QEMU:
 sudo apt-get install qemu-system-x86
 git clone https://github.com/cirosantilli/linux-kernel-module-cheat
 cd linux-kernel-module-cheat
-./download-latest-release
+./release-download-latest
 unzip lkmc-*.zip
 ./run --prebuilt
 ....
@@ -9404,22 +9404,7 @@ vim "$(./getvar --arch arm run_cmd_file)"
 ./"$(./getvar --arch arm run_cmd_file)"
 ....
 
-Next, you will also want to give the relevant images to save them time. Zip the images with:
-
-....
-./build-all -G
-./zip-img
-....
-
-Source: link:zip-img[]
-
-This generates a zip file:
-
-....
-out/lkmc-*.zip
-....
-
-which you can then upload somewhere, e.g. GitHub release assets as in https://github.com/cirosantilli/linux-kernel-module-cheat/releases/tag/test-replay-arm
+Next, you will also want to give the relevant images to save them time, see: <<zip-img>>.
 
 Finally, do a clone of the relevant repository out of tree and reproduce the bug there, to be 100% sure that it is an actual upstream bug, and to provide developers with the cleanest possible commands.
 
@@ -10338,7 +10323,7 @@ This can be used to check the determinism of:
 * <<norandmaps>>
 * <<qemu-record-and-replay>>
 
-==== Releases
+==== Release
 
 This is not yet super stable, but one day maybe this script will automatically do a release:
 
@@ -10348,8 +10333,46 @@ This is not yet super stable, but one day maybe this script will automatically d
 
 Source: link:release[].
 
+When ready, that script should:
+
+* build
+* test
+* package with <<release-zip>>
+* upload to GitHub with link:release-create-github[]
+
 This should in particular enable to easily update <<prebuilt>>.
 
+===== release-zip
+
+Create a zip containing all files required for <<prebuilt>>
+
+....
+./build-all -G
+./release-zip
+....
+
+Source: link:release-zip[]
+
+This generates a zip file:
+
+....
+echo "$(./getvar release_zip_file)"
+....
+
+which you can then upload somewhere.
+
+For example, you can create or update a GitHub release and upload automatically with:
+
+....
+git push
+printf "$GITHUB_TOKEN" > "$(./getvar github_token_file)"
+./release-upload
+....
+
+Source: link:release-upload[]
+
+TODO: generalize that so that people can upload to their forks.
+
 === Fairy tale
 
 ____

common.py

@@ -6,6 +6,7 @@
 import datetime
 import glob
 import imp
+import json
 import os
 import re
 import shlex
@@ -15,10 +16,10 @@
 import subprocess
 import sys
 import time
+import urllib
+import urllib.request
 
 this = sys.modules[__name__]
-
-# Default paths.
 root_dir = os.path.dirname(os.path.abspath(__file__))
 data_dir = os.path.join(root_dir, 'data')
 p9_dir = os.path.join(data_dir, '9p')
@@ -34,8 +35,7 @@
 qemu_src_dir = os.path.join(submodules_dir, 'qemu')
 parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark')
 ccache_dir = os.path.join('/usr', 'lib', 'ccache')
-
-# Other default variables.
+github_token_file = os.path.join(data_dir, 'github-token')
 arch_map = {
     'a': 'arm',
     'A': 'aarch64',
@@ -44,6 +44,9 @@
 arches = [arch_map[k] for k in arch_map]
 gem5_cpt_prefix = '^cpt\.'
 sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
+release_dir = os.path.join(this.out_dir, 'release')
+release_zip_file = os.path.join(this.release_dir, 'lkmc-{}.zip'.format(this.sha))
+github_repo_id = 'cirosantilli/linux-kernel-module-cheat'
 config_file = os.path.join(data_dir, 'config')
 if os.path.exists(config_file):
     config = imp.load_source('config', config_file)
@@ -189,6 +192,39 @@ def get_toolchain_tool(tool):
     global this
     return glob.glob(os.path.join(this.host_bin_dir, '*-buildroot-*-{}'.format(tool)))[0]
 
+def github_make_request(
+        authenticate=False,
+        data=None,
+        extra_headers=None,
+        path='',
+        subdomain='api',
+        url_params=None,
+        **extra_request_args
+    ):
+    global this
+    if extra_headers is None:
+        extra_headers = {}
+    headers = {'Accept': 'application/vnd.github.v3+json'}
+    headers.update(extra_headers)
+    if authenticate:
+        with open(this.github_token_file, 'r') as f:
+            token = f.read().rstrip()
+        headers['Authorization'] = 'token ' + token
+    if url_params is not None:
+        path += '?' + urllib.parse.urlencode(url_params)
+    request = urllib.request.Request(
+        'https://' + subdomain + '.github.com/repos/' + github_repo_id + path,
+        headers=headers,
+        data=data,
+        **extra_request_args
+    )
+    response_body = urllib.request.urlopen(request).read().decode()
+    if response_body:
+        _json = json.loads(response_body)
+    else:
+        _json = {}
+    return _json
+
 def log_error(msg):
     print('error: {}'.format(msg), file=sys.stderr)
 
@@ -340,7 +376,7 @@ def run_cmd(
     #signal.signal(signal.SIGPIPE, sigpipe_old)
     return proc.returncode
 
-def setup(parser, **extra_args):
+def setup(parser):
     '''
     Parse the command line arguments, and setup several variables based on them.
     Typically done after getting inputs from the command line arguments.

release

@@ -1,31 +1,27 @@
 #!/usr/bin/env python3
 
+'''
+https://upload.com/cirosantilli/linux-kernel-module-cheat#release
+'''
+
 import imp
 import os
 import subprocess
 
 import common
-zip_img = imp.load_source('zip_img', os.path.join(common.root_dir, 'zip-img'))
+release_zip = imp.load_source('release_zip', os.path.join(common.root_dir, 'release-zip'))
+release_upload = imp.load_source('release_upload', os.path.join(common.root_dir, 'release-upload'))
+
+# TODO factor those out so we don't redo the same thing multiple times.
+# subprocess.check_call([os.path.join(common.root_dir, 'test')])
+# subprocess.check_call([os.path.join(common.root_dir, ''bench-all', '-A', '-u'])
 
-subprocess.check_call([os.path.join(common.root_dir, 'test')])
 # A clean release requires a full rebuild unless we hack it :-(
 # We can't just use our curent build as it contains packages we've
 # installed in random experiments. And with EXT2: we can't easily
 # know what the smallest root filesystem size is and use it either...
 # https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot
 subprocess.check_call([os.path.join(common.root_dir, 'build-all')])
-zip_img.main()
-tag = 'sha-{}'.format(common.sha)
-subprocess.check_call(['git', 'tag', '-f', tag])
-subprocess.check_call(['git', 'push', '--tags'])
-# TODO
-# - https://stackoverflow.com/questions/41022470/curl-request-to-add-file-to-github-release
-# - https://stackoverflow.com/questions/38627115/upload-files-to-github-directory-using-github-api
-# upload_basename = 'images-{}.zip'.format(common.sha)
-#curl "https://api.github.com/repos/cirosantilli/linux-kernel-module-cheat/releases/tags/${tag}/assets?access_token=$(cat data/access_token)&tag_name=${upload_basename}" \
-#  --header 'Content-Type: application/zip' \
-#  --upload-file "${common_out_dir}/${upload_basename}" \
-#  -H 'Accept: application/vnd.github.v3+json' \
-#  -X POST \
-#;
-subprocess.check_call(['./bench-all', '-A', '-u'])
+release_zip.main()
+subprocess.check_call(['git', 'push'])
+release_upload.main()

download-latest-release renamed to release-download-latest

@@ -1,16 +1,16 @@
 #!/usr/bin/env python3
 
 '''
-Download the latest release from GitHub:
+Usage: https://github.com/cirosantilli/linux-kernel-module-cheat#prebuilt
+
+Implementation:
 https://stackoverflow.com/questions/24987542/is-there-a-link-to-github-for-downloading-a-file-in-the-latest-release-of-a-repo/50540591#50540591
 '''
 
-import json
 import urllib.request
 
-_json = json.loads(urllib.request.urlopen(urllib.request.Request(
-    'https://api.github.com/repos/cirosantilli/linux-kernel-module-cheat/releases',
-    headers={'Accept' : 'application/vnd.github.full+json"text/html'}
-)).read())
+import common
+
+_json = common.github_make_request(path='/releases')
 asset = _json[0]['assets'][0]
 urllib.request.urlretrieve(asset['browser_download_url'], asset['name'])

release-upload

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+'''
+Usage: https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip
+
+Implementation:
+
+* https://stackoverflow.com/questions/5207269/how-to-release-a-build-artifact-asset-on-github-with-a-script/52354732#52354732
+* https://stackoverflow.com/questions/38153418/can-someone-give-a-python-requests-example-of-uploading-a-release-asset-in-githu/52354681#52354681
+'''
+
+import json
+import os
+import sys
+
+import urllib.error
+
+import common
+
+def main():
+    repo = common.github_repo_id
+    tag = 'sha-{}'.format(common.sha)
+    upload_path = common.release_zip_file
+
+    # Check the release already exists.
+    try:
+        _json = common.github_make_request(path='/releases/tags/' + tag)
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            release_exists = False
+        else:
+            raise e
+    else:
+        release_exists = True
+        release_id = _json['id']
+
+    # Create release if not yet created.
+    if not release_exists:
+        _json = common.github_make_request(
+            authenticate=True,
+            data=json.dumps({
+                'tag_name': tag,
+                'name': tag,
+                'prerelease': True,
+            }).encode(),
+            path='/releases'
+        )
+        release_id = _json['id']
+
+    asset_name = os.path.split(upload_path)[1]
+
+    # Clear the prebuilts for a upload.
+    _json = common.github_make_request(
+        path=('/releases/' + str(release_id) + '/assets'),
+    )
+    for asset in _json:
+        if asset['name'] == asset_name:
+            _json = common.github_make_request(
+                authenticate=True,
+                path=('/releases/assets/' + str(asset['id'])),
+                method='DELETE',
+            )
+            break
+
+    # Upload the prebuilt.
+    with open(upload_path, 'br') as myfile:
+        content = myfile.read()
+    _json = common.github_make_request(
+        authenticate=True,
+        data=content,
+        extra_headers={'Content-Type': 'application/zip'},
+        path=('/releases/' + str(release_id) + '/assets'),
+        subdomain='uploads',
+        url_params={'name': asset_name},
+    )
+
+if __name__ == '__main__':
+    main()

zip-img renamed to release-zip

@@ -1,16 +1,20 @@
 #!/usr/bin/env python3
 
+'''
+https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip
+'''
+
 import os
 import subprocess
 import zipfile
 
 import common
 
 def main():
-    outfile = os.path.join(common.out_dir, 'lkmc-{}.zip'.format(common.sha))
-    if os.path.exists(outfile):
-        os.unlink(outfile)
-    zipf = zipfile.ZipFile(outfile, 'w', zipfile.ZIP_DEFLATED)
+    os.makedirs(common.release_dir, exist_ok=True)
+    if os.path.exists(common.release_zip_file):
+        os.unlink(common.release_zip_file)
+    zipf = zipfile.ZipFile(common.release_zip_file, 'w', zipfile.ZIP_DEFLATED)
     for arch in common.arches:
         common.setup(common.get_argparse(default_args={'arch': arch}))
         zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir))