Cleanup sysroot_scripts

- Remove UpdatePackageLists* since this functionality was already performed by
  BuildSysroot*.
- Remove the "Revision" key from sysroots.json since the "Sha1Sum" key is enough
  to uniquely identify each sysroot.
- Move packagelist.* into generated_package_lists/*.  There were at least 2
  occasions where folks tried to manually update these files.

R=thestig
BUG=None

Change-Id: Id7f8ea9ade6b2f445997f445412ec2120b3f7770
Reviewed-on: https://chromium-review.googlesource.com/c/1268942
Reviewed-by: Lei Zhang <thestig@chromium.org>
Commit-Queue: Thomas Anderson <thomasanderson@chromium.org>
Cr-Commit-Position: refs/heads/master@{#598043}

build/linux/sysroot_scripts/build_and_upload.py

@@ -35,14 +35,10 @@ def get_proc_output(args):
   return subprocess.check_output(args).strip()
 
 def build_and_upload(script_path, distro, release, arch, lock):
-  # TODO(thomasanderson):  Find out which revision 'git-cl upload' uses to
-  # calculate the diff against and use that instead of HEAD.
   script_dir = os.path.dirname(os.path.realpath(__file__))
-  revision = get_proc_output(['git', '-C', script_dir, 'rev-parse', 'HEAD'])
 
-  run_script([script_path, 'UpdatePackageLists%s' % arch])
+  run_script([script_path, 'BuildSysroot' + arch])
-  run_script([script_path, 'BuildSysroot%s' % arch])
+  run_script([script_path, 'UploadSysroot' + arch])
-  run_script([script_path, 'UploadSysroot%s' % arch, revision])
 
   tarball = '%s_%s_%s_sysroot.tar.xz' % (distro, release, arch.lower())
   tarxz_path = os.path.join(script_dir, "..", "..", "..", "out",
@@ -51,10 +47,9 @@ def build_and_upload(script_path, distro, release, arch, lock):
   sysroot_dir = '%s_%s_%s-sysroot' % (distro, release, arch.lower())
 
   sysroot_metadata = {
-      'Revision': revision,
       'Tarball': tarball,
       'Sha1Sum': sha1sum,
-      'SysrootDir': sysroot_dir
+      'SysrootDir': sysroot_dir,
   }
   with lock:
     with open(os.path.join(script_dir, 'sysroots.json'), 'rw+') as f:

build/linux/sysroot_scripts/install-sysroot.py

@@ -106,7 +106,6 @@ def GetSysrootDict(target_platform, target_arch):
 
 def InstallSysroot(target_platform, target_arch):
   sysroot_dict = GetSysrootDict(target_platform, target_arch)
-  revision = sysroot_dict['Revision']
   tarball_filename = sysroot_dict['Tarball']
   tarball_sha1sum = sysroot_dict['Sha1Sum']
   # TODO(thestig) Consider putting this elsewhere to avoid having to recreate
@@ -114,7 +113,8 @@ def InstallSysroot(target_platform, target_arch):
   linux_dir = os.path.dirname(SCRIPT_DIR)
   sysroot = os.path.join(linux_dir, sysroot_dict['SysrootDir'])
 
-  url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, revision, tarball_filename)
+  url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, tarball_sha1sum,
+                         tarball_filename)
 
   stamp = os.path.join(sysroot, '.stamp')
   if os.path.exists(stamp):

build/linux/sysroot_scripts/sysroot-creator.sh

@@ -61,12 +61,12 @@ readonly PACKAGES_EXT=xz
 readonly RELEASE_FILE="Release"
 readonly RELEASE_FILE_GPG="Release.gpg"
 
-readonly DEBIAN_DEP_LIST_AMD64="packagelist.${DIST}.amd64"
+readonly DEBIAN_DEP_LIST_AMD64="generated_package_lists/${DIST}.amd64"
-readonly DEBIAN_DEP_LIST_I386="packagelist.${DIST}.i386"
+readonly DEBIAN_DEP_LIST_I386="generated_package_lists/${DIST}.i386"
-readonly DEBIAN_DEP_LIST_ARM="packagelist.${DIST}.arm"
+readonly DEBIAN_DEP_LIST_ARM="generated_package_lists/${DIST}.arm"
-readonly DEBIAN_DEP_LIST_ARM64="packagelist.${DIST}.arm64"
+readonly DEBIAN_DEP_LIST_ARM64="generated_package_lists/${DIST}.arm64"
-readonly DEBIAN_DEP_LIST_MIPS="packagelist.${DIST}.mipsel"
+readonly DEBIAN_DEP_LIST_MIPS="generated_package_lists/${DIST}.mipsel"
-readonly DEBIAN_DEP_LIST_MIPS64EL="packagelist.${DIST}.mips64el"
+readonly DEBIAN_DEP_LIST_MIPS64EL="generated_package_lists/${DIST}.mips64el"
 
 
 ######################################################################
@@ -265,17 +265,6 @@ StripChecksumsFromPackageList() {
   sed -i 's/ [a-f0-9]\{64\}$//' "$package_file"
 }
 
-VerifyPackageFilesMatch() {
-  local downloaded_package_file="$1"
-  local stored_package_file="$2"
-  diff -u "$downloaded_package_file" "$stored_package_file"
-  if [ "$?" -ne "0" ]; then
-    echo "ERROR: downloaded package files does not match $2."
-    echo "You may need to run UpdatePackageLists."
-    exit 1
-  fi
-}
-
 ######################################################################
 #
 ######################################################################
@@ -500,11 +489,10 @@ BuildSysrootAmd64() {
     return
   fi
   ClearInstallDir
-  local package_file="$BUILD_DIR/package_with_sha256sum_amd64"
+  local package_file="${DEBIAN_DEP_LIST_AMD64}"
   GeneratePackageListAmd64 "$package_file"
   local files_and_sha256sums="$(cat ${package_file})"
   StripChecksumsFromPackageList "$package_file"
-  VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_AMD64"
   InstallIntoSysroot ${files_and_sha256sums}
   CleanupJailSymlinks
   HacksAndPatchesAmd64
@@ -521,11 +509,10 @@ BuildSysrootI386() {
     return
   fi
   ClearInstallDir
-  local package_file="$BUILD_DIR/package_with_sha256sum_i386"
+  local package_file="${DEBIAN_DEP_LIST_I386}"
   GeneratePackageListI386 "$package_file"
   local files_and_sha256sums="$(cat ${package_file})"
   StripChecksumsFromPackageList "$package_file"
-  VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_I386"
   InstallIntoSysroot ${files_and_sha256sums}
   CleanupJailSymlinks
   HacksAndPatchesI386
@@ -542,11 +529,10 @@ BuildSysrootARM() {
     return
   fi
   ClearInstallDir
-  local package_file="$BUILD_DIR/package_with_sha256sum_arm"
+  local package_file="${DEBIAN_DEP_LIST_ARM}"
   GeneratePackageListARM "$package_file"
   local files_and_sha256sums="$(cat ${package_file})"
   StripChecksumsFromPackageList "$package_file"
-  VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_ARM"
   InstallIntoSysroot ${files_and_sha256sums}
   CleanupJailSymlinks
   HacksAndPatchesARM
@@ -563,11 +549,10 @@ BuildSysrootARM64() {
     return
   fi
   ClearInstallDir
-  local package_file="$BUILD_DIR/package_with_sha256sum_arm64"
+  local package_file="${DEBIAN_DEP_LIST_ARM64}"
   GeneratePackageListARM64 "$package_file"
   local files_and_sha256sums="$(cat ${package_file})"
   StripChecksumsFromPackageList "$package_file"
-  VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_ARM64"
   InstallIntoSysroot ${files_and_sha256sums}
   CleanupJailSymlinks
   HacksAndPatchesARM64
@@ -575,7 +560,6 @@ BuildSysrootARM64() {
   CreateTarBall
 }
 
-
 #@
 #@ BuildSysrootMips
 #@
@@ -585,11 +569,10 @@ BuildSysrootMips() {
     return
   fi
   ClearInstallDir
-  local package_file="$BUILD_DIR/package_with_sha256sum_mips"
+  local package_file="${DEBIAN_DEP_LIST_MIPS}"
   GeneratePackageListMips "$package_file"
   local files_and_sha256sums="$(cat ${package_file})"
   StripChecksumsFromPackageList "$package_file"
-  VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_MIPS"
   InstallIntoSysroot ${files_and_sha256sums}
   CleanupJailSymlinks
   HacksAndPatchesMips
@@ -597,7 +580,6 @@ BuildSysrootMips() {
   CreateTarBall
 }
 
-
 #@
 #@ BuildSysrootMips64el
 #@
@@ -607,11 +589,10 @@ BuildSysrootMips64el() {
     return
   fi
   ClearInstallDir
-  local package_file="$BUILD_DIR/package_with_sha256sum_mips64el"
+  local package_file="${DEBIAN_DEP_LIST_MIPS64EL}"
   GeneratePackageListMips64el "$package_file"
   local files_and_sha256sums="$(cat ${package_file})"
   StripChecksumsFromPackageList "$package_file"
-  VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_MIPS64EL"
   InstallIntoSysroot ${files_and_sha256sums}
   CleanupJailSymlinks
   HacksAndPatchesMips64el
@@ -633,19 +614,15 @@ BuildSysrootAll() {
 }
 
 UploadSysroot() {
-  local rev=$1
+  local sha=$(sha1sum "${TARBALL}" | awk '{print $1;}')
-  if [ -z "${rev}" ]; then
-    echo "Please specify a revision to upload at."
-    exit 1
-  fi
   set -x
   gsutil cp -a public-read "${TARBALL}" \
-      "gs://chrome-linux-sysroot/toolchain/$rev/"
+      "gs://chrome-linux-sysroot/toolchain/$sha/"
   set +x
 }
 
 #@
-#@ UploadSysrootAmd64 <revision>
+#@ UploadSysrootAmd64
 #@
 UploadSysrootAmd64() {
   if [ "$HAS_ARCH_AMD64" = "0" ]; then
@@ -655,7 +632,7 @@ UploadSysrootAmd64() {
 }
 
 #@
-#@ UploadSysrootI386 <revision>
+#@ UploadSysrootI386
 #@
 UploadSysrootI386() {
   if [ "$HAS_ARCH_I386" = "0" ]; then
@@ -665,7 +642,7 @@ UploadSysrootI386() {
 }
 
 #@
-#@ UploadSysrootARM <revision>
+#@ UploadSysrootARM
 #@
 UploadSysrootARM() {
   if [ "$HAS_ARCH_ARM" = "0" ]; then
@@ -675,7 +652,7 @@ UploadSysrootARM() {
 }
 
 #@
-#@ UploadSysrootARM64 <revision>
+#@ UploadSysrootARM64
 #@
 UploadSysrootARM64() {
   if [ "$HAS_ARCH_ARM64" = "0" ]; then
@@ -685,7 +662,7 @@ UploadSysrootARM64() {
 }
 
 #@
-#@ UploadSysrootMips <revision>
+#@ UploadSysrootMips
 #@
 UploadSysrootMips() {
   if [ "$HAS_ARCH_MIPS" = "0" ]; then
@@ -695,7 +672,7 @@ UploadSysrootMips() {
 }
 
 #@
-#@ UploadSysrootMips64el <revision>
+#@ UploadSysrootMips64el
 #@
 UploadSysrootMips64el() {
   if [ "$HAS_ARCH_MIPS64EL" = "0" ]; then
@@ -705,7 +682,7 @@ UploadSysrootMips64el() {
 }
 
 #@
-#@ UploadSysrootAll <revision>
+#@ UploadSysrootAll
 #@
 #@    Upload sysroot image for all architectures
 UploadSysrootAll() {
@@ -802,98 +779,6 @@ GeneratePackageList() {
   sort "$output_file" -o "$output_file"
 }
 
-#@
-#@ UpdatePackageListsAmd64
-#@
-#@     Regenerate the package lists such that they contain an up-to-date
-#@     list of URLs within the Debian archive. (For amd64)
-UpdatePackageListsAmd64() {
-  if [ "$HAS_ARCH_AMD64" = "0" ]; then
-    return
-  fi
-  GeneratePackageListAmd64 "$DEBIAN_DEP_LIST_AMD64"
-  StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_AMD64"
-}
-
-#@
-#@ UpdatePackageListsI386
-#@
-#@     Regenerate the package lists such that they contain an up-to-date
-#@     list of URLs within the Debian archive. (For i386)
-UpdatePackageListsI386() {
-  if [ "$HAS_ARCH_I386" = "0" ]; then
-    return
-  fi
-  GeneratePackageListI386 "$DEBIAN_DEP_LIST_I386"
-  StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_I386"
-}
-
-#@
-#@ UpdatePackageListsARM
-#@
-#@     Regenerate the package lists such that they contain an up-to-date
-#@     list of URLs within the Debian archive. (For arm)
-UpdatePackageListsARM() {
-  if [ "$HAS_ARCH_ARM" = "0" ]; then
-    return
-  fi
-  GeneratePackageListARM "$DEBIAN_DEP_LIST_ARM"
-  StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_ARM"
-}
-
-#@
-#@ UpdatePackageListsARM64
-#@
-#@     Regenerate the package lists such that they contain an up-to-date
-#@     list of URLs within the Debian archive. (For arm64)
-UpdatePackageListsARM64() {
-  if [ "$HAS_ARCH_ARM64" = "0" ]; then
-    return
-  fi
-  GeneratePackageListARM64 "$DEBIAN_DEP_LIST_ARM64"
-  StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_ARM64"
-}
-
-#@
-#@ UpdatePackageListsMips
-#@
-#@     Regenerate the package lists such that they contain an up-to-date
-#@     list of URLs within the Debian archive. (For mips)
-UpdatePackageListsMips() {
-  if [ "$HAS_ARCH_MIPS" = "0" ]; then
-    return
-  fi
-  GeneratePackageListMips "$DEBIAN_DEP_LIST_MIPS"
-  StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_MIPS"
-}
-
-
-#@
-#@ UpdatePackageListsMips64el
-#@
-#@     Regenerate the package lists such that they contain an up-to-date
-#@     list of URLs within the Debian archive. (For mips64el)
-UpdatePackageListsMips64el() {
-  if [ "$HAS_ARCH_MIPS64EL" = "0" ]; then
-    return
-  fi
-  GeneratePackageListMips64el "$DEBIAN_DEP_LIST_MIPS64EL"
-  StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_MIPS64EL"
-}
-
-#@
-#@ UpdatePackageListsAll
-#@
-#@    Regenerate the package lists for all architectures.
-UpdatePackageListsAll() {
-  RunCommand UpdatePackageListsAmd64
-  RunCommand UpdatePackageListsI386
-  RunCommand UpdatePackageListsARM
-  RunCommand UpdatePackageListsARM64
-  RunCommand UpdatePackageListsMips
-  RunCommand UpdatePackageListsMips64el
-}
-
 #@
 #@ PrintArchitectures
 #@

build/linux/sysroot_scripts/sysroots.json

@@ -1,36 +1,30 @@
 {
     "sid_amd64": {
-        "Revision": "5758428cde52994cc3e383e603fbd88fe321d42c",
         "Sha1Sum": "e7c53f04bd88d29d075bfd1f62b073aeb69cbe09",
         "SysrootDir": "debian_sid_amd64-sysroot",
         "Tarball": "debian_sid_amd64_sysroot.tar.xz"
     },
     "sid_arm": {
-        "Revision": "5758428cde52994cc3e383e603fbd88fe321d42c",
         "Sha1Sum": "ef5c4f84bcafb7a3796d36bb1db7826317dde51c",
         "SysrootDir": "debian_sid_arm-sysroot",
         "Tarball": "debian_sid_arm_sysroot.tar.xz"
     },
     "sid_arm64": {
-        "Revision": "5758428cde52994cc3e383e603fbd88fe321d42c",
         "Sha1Sum": "953c2471bc7e71a788309f6c2d2003e8b703305d",
         "SysrootDir": "debian_sid_arm64-sysroot",
         "Tarball": "debian_sid_arm64_sysroot.tar.xz"
     },
     "sid_i386": {
-        "Revision": "5758428cde52994cc3e383e603fbd88fe321d42c",
         "Sha1Sum": "9e6279438ece6fb42b5333ca90d5e9d0c188a403",
         "SysrootDir": "debian_sid_i386-sysroot",
         "Tarball": "debian_sid_i386_sysroot.tar.xz"
     },
     "sid_mips": {
-        "Revision": "5758428cde52994cc3e383e603fbd88fe321d42c",
         "Sha1Sum": "958731a68a169631c0450efb15410ccc4135ef2a",
         "SysrootDir": "debian_sid_mips-sysroot",
         "Tarball": "debian_sid_mips_sysroot.tar.xz"
     },
     "sid_mips64el": {
-        "Revision": "5758428cde52994cc3e383e603fbd88fe321d42c",
         "Sha1Sum": "51ca1f4092ac76ad1a1da953f0f3ce1aea947a42",
         "SysrootDir": "debian_sid_mips64el-sysroot",
         "Tarball": "debian_sid_mips64el_sysroot.tar.xz"

docs/linux_sysroot.md

@@ -29,17 +29,7 @@ new packages, skip to `Using build_and_upload.py`.
 ### Adding new packages
 
 To add a new package, edit the `sysroot-creator-*.sh` scripts and modify the
-`DEBIAN_PACKAGES` list, then run the update step (`UpdatePackageListsAll`).  The
+`DEBIAN_PACKAGES` list.
-steps below will use `sysroot-creator-stretch.sh` as an example.
-
-### Updating existing package list
-
-To update packages to the latest versions run:
-
-    $ ./sysroot-creator-stretch.sh UpdatePackageListsAll
-
-This command will update the package lists that are stored alongside the script.
-If no packages have changed then this script will have no effect.
 
 ### Rebuilding
 
@@ -61,7 +51,7 @@ the one on Google Cloud Storage.
 
 To upload images to Google Cloud Storage run the following command:
 
-    $ ./sysroot-creator-stretch.sh UploadSysrootAll <SHA1>
+    $ ./sysroot-creator-stretch.sh UploadSysrootAll
 
 Here you should use the SHA1 of the git revision at which the images were
 created.