Generalize git pushing in a bash function

GNUmakefile

@@ -63,6 +63,21 @@ else
 DIST_DATE_ARG=
 endif
 
+PUB_GIT?=1
+PUB_WEBSITE?=1
+
+ifeq ($(PUB_GIT),1)
+PUB_GIT_ARG=-g
+else
+PUB_GIT_ARG=
+endif
+
+ifeq ($(PUB_WEBSITE),1)
+PUB_WEBSITE_ARG=-g
+else
+PUB_WEBSITE_ARG=
+endif
+
 export GO_BUILD_TAG
 export UI_BUILD_TAG
 export UI_LEGACY_BUILD_TAG
@@ -105,7 +120,7 @@ dist:
 	@$(SHELL) $(CURDIR)/build-support/scripts/release.sh -t '$(DIST_TAG)' -b '$(DIST_BUILD)' -S '$(DIST_SIGN)' '$(DIST_VERSION_ARG)' '$(DIST_DATE_ARG)'
 	
 publish:
-	@$(SHELL) $(CURDIR)/build-support/scripts/publish.sh -g -w
+	@$(SHELL) $(CURDIR)/build-support/scripts/publish.sh '$(PUB_GIT_ARG)' '$(PUB_WEBSITE_ARG)'
 
 dev-tree:
 	@$(SHELL) $(CURDIR)/build-support/scripts/dev.sh

build-support/functions/01-util.sh

@@ -313,7 +313,7 @@ function normalize_git_url {
    url="${1#https://}"
    url="${url#git@}"
    url="${url%.git}"
-   url="$(sed ${SED_EXT} -e 's/\([^\/:]*\)[:\/]\(.*\)/\1:\2/' <<< "${url}")"
+   url="$(sed ${SED_EXT} -e 's/([^\/:]*)[:\/](.*)/\1:\2/' <<< "${url}")"
    echo "$url"
    return 0
 }
@@ -336,6 +336,7 @@ function find_git_remote {
    fi
    
    need_url=$(normalize_git_url "${PUBLISH_GIT_HOST}:${PUBLISH_GIT_REPO}")
+   debug "Required normalized remote: ${need_url}"
    
    pushd "$1" > /dev/null
    
@@ -345,6 +346,7 @@ function find_git_remote {
       url=$(git remote get-url --push ${remote}) || continue
       url=$(normalize_git_url "${url}")
       
+      debug "Testing Remote: ${remote}: ${url}"
       if test "${url}" == "${need_url}"
       then
          echo "${remote}"
@@ -354,7 +356,7 @@ function find_git_remote {
    done
    
    popd > /dev/null
-   return $ret
+   return ${ret}
 }
 
 function is_git_clean {
@@ -390,6 +392,113 @@ function is_git_clean {
    return ${ret}
 }
 
+function update_git_env {
+   # Arguments:
+   #   $1 - Path to git repo
+   #
+   # Returns:
+   #   0 - success
+   #   * - error
+   #
+   
+   if ! test -d "$1"
+   then
+      err "ERROR: '$1' is not a directory. is_git_clean must be called with the path to a git repo as the first argument'" 
+      return 1
+   fi
+   
+   export GIT_COMMIT=$(git rev-parse --short HEAD)
+   export GIT_DIRTY=$(test -n "$(git status --porcelain)" && echo "+CHANGES")
+   export GIT_DESCRIBE=$(git describe --tags --always)
+   export GIT_IMPORT=github.com/hashicorp/consul/version
+   export GOLDFLAGS="-X ${GIT_IMPORT}.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X ${GIT_IMPORT}.GitDescribe=${GIT_DESCRIBE}"
+   return 0
+}
+
+function git_push_ref {
+   # Arguments:
+   #   $1 - Path to the top level Consul source
+   #   $2 - Git ref (optional)
+   #
+   # Returns:
+   #   0 - success
+   #   * - error
+   
+   if ! test -d "$1"
+   then
+      err "ERROR: '$1' is not a directory. push_git_release must be called with the path to the top level source as the first argument'" 
+      return 1
+   fi
+   
+   local sdir="$1"
+   local ret=0
+   
+   # find the correct remote corresponding to the desired repo (basically prevent pushing enterprise to oss or oss to enterprise)
+   local remote=$(find_git_remote "${sdir}") || return 1
+   status "Using git remote: ${remote}"
+   
+   local ref=""
+   
+   pushd "${sdir}" > /dev/null
+   
+   if test -z "$2"
+   then
+      # If no git ref was provided we lookup the current local branch and its tracking branch
+      # It must have a tracking upstream and it must be tracking the sanctioned git remote
+      local head=$(git_branch "${sdir}") || return 1
+      local upstream=$(git_upstream "${sdir}") || return 1
+   
+      # upstream branch for this branch does not track the remote we need to push to
+      # basically this checks that the upstream (could be something like origin/master) references the correct remote
+      # if it doesn't then the string modification wont apply and the var will reamin unchanged and equal to itself.
+      if test "${upstream#${remote}/}" == "${upstream}"
+      then
+         err "ERROR: Upstream branch '${upstream}' does not track the correct remote '${remote}' - cannot push"
+         ret=1
+      fi
+      ref="refs/heads/${head}"
+   else
+      # A git ref was provided - get the full ref and make sure it isn't ambiguous and also to
+      # be able to determine whether its a branch or tag we are pushing
+      ref_out=$(git rev-parse --symbolic-full-name "$2" --)
+      
+      # -ne 2 because it should have the ref on one line followed by a line with '--'
+      if test "$(wc -l <<< "${ref_out}")" -ne 2
+      then
+         err "ERROR: Git ref '$2' is ambiguous"
+         debug "${ref_out}"
+         ret=1
+      else
+         ref=$(head -n 1 <<< "${ref_out}")   
+      fi
+   fi
+   
+   if test ${ret} -eq 0
+   then
+      case "${ref}" in
+         refs/tags/*)
+            status "Pushing tag ${ref#refs/tags/} to ${remote}"
+            ;;
+         refs/heads/*)
+            status "Pushing local branch ${ref#refs/tags/} to ${remote}"
+            ;;
+         *)
+            err "ERROR: git_push_ref func is refusing to push ref that isn't a branch or tag"
+            return 1
+      esac
+   
+      if ! git push "${remote}" "${ref}"
+      then
+         err "ERROR: Failed to push ${ref} to remote: ${remote}"
+         ret=1
+      fi
+   fi
+   
+   popd > /dev/null
+   
+   return $ret
+}
+
 function update_version {
    # Arguments:
    #   $1 - Path to the version file
@@ -578,3 +687,70 @@ function set_dev_mode {
    
    return 0
 }
+
+function git_staging_empty {
+   # Arguments:
+   #   $1 - Path to git repo
+   #
+   # Returns:
+   #   0 - success (nothing staged)
+   #   * - error (staged files)
+   
+   if ! test -d "$1"
+   then
+      err "ERROR: '$1' is not a directory. commit_dev_mode must be called with the path to a git repo as the first argument'" 
+      return 1
+   fi
+   
+   pushd "$1" > /dev/null
+   
+   declare -i ret=0
+   
+   for status in $(git status --porcelain=v2 | awk '{print $2}' | cut -b 1)
+   do
+      if test "${status}" != "."
+      then 
+         ret=1
+         break
+      fi
+   done
+   
+   popd > /dev/null
+   return ${ret}   
+}
+
+function commit_dev_mode {
+   # Arguments:
+   #   $1 - Path to top level Consul source
+   #
+   # Returns:
+   #   0 - success
+   #   * - error
+   
+   if ! test -d "$1"
+   then
+      err "ERROR: '$1' is not a directory. commit_dev_mode must be called with the path to a git repo as the first argument'" 
+      return 1
+   fi
+   
+   status "Checking for previously staged files"
+   git_staging_empty "$1" || return 1
+   
+   declare -i ret=0
+   
+   pushd "$1" > /dev/null
+   
+   status "Staging CHANGELOG.md and version_*.go files"
+   git add CHANGELOG.md && git add version/version_*.go
+   ret=$?
+   
+   if test ${ret} -eq 0
+   then
+      status "Adding Commit"
+      git commit -m "Putting source back into Dev Mode"
+      ret=$?      
+   fi
+   
+   popd >/dev/null
+   return ${ret}
+}

build-support/functions/03-release.sh

@@ -258,8 +258,6 @@ function build_consul_release {
    build_consul "$1" "" "$2"  
 }
 
-
-
 function build_release {
    # Arguments: (yeah there are lots)
    #   $1 - Path to the top level Consul source
@@ -328,7 +326,7 @@ function build_release {
       set_vers=$(get_version "${sdir}" false false)
    fi
    
-   if ! set_release_mode "${sdir}" "${set_vers}" "${set_date}"
+   if is_set "${do_tag}" && ! set_release_mode "${sdir}" "${set_vers}" "${set_date}"
    then
       err "ERROR: Failed to put source into release mode"
       return 1 
@@ -398,6 +396,8 @@ function build_release {
          err "ERROR: Failed to tag the release" 
          return 1
       fi
+      
+      update_git_env "${sdir}"
    fi
    
    if is_set "${do_build}"

build-support/functions/04-publish.sh

@@ -19,59 +19,6 @@ function hashicorp_release {
    return 0
 }
 
-function push_git_release {
-   # Arguments:
-   #   $1 - Path to the top level Consul source
-   #   $2 - Tag to push
-   #
-   # Returns:
-   #   0 - success
-   #   * - error
-   
-   if ! test -d "$1"
-   then
-      err "ERROR: '$1' is not a directory. push_git_release must be called with the path to the top level source as the first argument'" 
-      return 1
-   fi
-   
-   local sdir="$1"
-   local ret=0
-   
-   # find the correct remote corresponding to the desired repo (basically prevent pushing enterprise to oss or oss to enterprise)
-   local remote=$(find_git_remote "${sdir}") || return 1
-   local head=$(git_branch "${sdir}") || return 1
-   local upstream=$(git_upstream "${sdir}") || return 1
-   status "Using git remote: ${remote}"
-   
-   # upstream branch for this branch does not track the remote we need to push to
-   if test "${upstream#${remote}}" == "${upstream}"
-   then
-      err "ERROR: Upstream branch '${upstream}' does not track the correct remote '${remote}'"
-      return 1
-   fi
-   
-   pushd "${sdir}" > /dev/null
-   
-   status "Pushing local branch ${head} to ${upstream}"
-   if ! git push "${remote}"
-   then
-      err "ERROR: Failed to push to remote: ${remote}"
-      ret=1
-   fi
-   
-   status "Pushing tag ${2} to ${remote}"
-   if test "${ret}" -eq 0 && ! git push "${remote}" "${2}"
-   then
-      err "ERROR: Failed to push tag ${2} to ${remote}"
-      ret = 1
-   fi
-   
-   popd > /dev/null
-   
-   
-   return $ret
-}
-
 function confirm_git_push_changes {
    # Arguments:
    #   $1 - Path to git repo
@@ -234,7 +181,8 @@ function publish_release {
    if is_set "${pub_git}"
    then
       status_stage "==> Pushing to Git"
-      push_git_release "$1" "v${vers}" || return 1
+      git_push_ref "$1" || return 1
+      git_push_ref "$1" "v${vers}" || return 1
    fi
    
    if is_set "${pub_hc_releases}"

build-support/scripts/build-docker.sh

@@ -16,6 +16,10 @@ function usage {
 cat <<-EOF
 Usage: ${SCRIPT_NAME} (consul|ui|ui-legacy|static-assets) [<options ...>]
 
+Description:
+   This script will build the various Consul components within docker containers
+   and copy all the relevant artifacts out of the containers back to the source.
+
 Options:
    -i | --image      IMAGE       Alternative Docker image to run the build within.
                                
@@ -38,10 +42,7 @@ function main {
    declare    image=
    declare    sdir="${SOURCE_DIR}"
    declare -i refresh=0
-   declare    command="$1"
+   declare    command=""
-   
-   # get rid of the subcommand
-   shift
    
    while test $# -gt 0
    do
@@ -80,6 +81,10 @@ function main {
             refresh=1
             shift
             ;;
+         consul | ui | ui-legacy | static-assets )
+            command="$1"
+            shift
+            ;;
          * )
             err_usage "ERROR: Unknown argument '$1'"
             return 1
@@ -87,6 +92,12 @@ function main {
       esac
    done
    
+   if test -z "${command}"
+   then
+      err_usage "ERROR: No command specified"
+      return 1
+   fi
+   
    case "${command}" in 
       consul )
          if is_set "${refresh}"

build-support/scripts/build-local.sh

@@ -16,6 +16,11 @@ function usage {
 cat <<-EOF
 Usage: ${SCRIPT_NAME} [<options ...>]
 
+Description:
+   This script will build the Consul binary on the local system.
+   All the requisite tooling must be installed for this to be
+   successful.
+
 Options:
                        
    -s | --source     DIR         Path to source to build.

build-support/scripts/dev.sh

@@ -16,11 +16,18 @@ function usage {
 cat <<-EOF
 Usage: ${SCRIPT_NAME} [<options ...>]
 
+Description:
+
+   This script will put the source back into dev mode after a release.
+
 Options:
                        
    -s | --source     DIR         Path to source to build.
                                  Defaults to "${SOURCE_DIR}"
                                  
+   --no-git                      Do not commit or attempt to push
+                                 the changes back to the upstream.
+                                 
    -h | --help                   Print this help text.
 EOF
 }
@@ -32,9 +39,10 @@ function err_usage {
 }
 
 function main {
-   declare sdir="${SOURCE_DIR}"
+   declare    sdir="${SOURCE_DIR}"
-   declare build_os=""
+   declare    build_os=""
-   declare build_arch=""
+   declare    build_arch=""
+   declare -i do_git=1
    
    
    while test $# -gt 0
@@ -60,6 +68,10 @@ function main {
             sdir="$2"
             shift 2
             ;;
+         --no-git )
+            do_git=0
+            shift
+            ;;
          * )
             err_usage "ERROR: Unknown argument: '$1'"
             return 1
@@ -69,6 +81,18 @@ function main {
    
    set_dev_mode "${sdir}" || return 1
    
+   if is_set "${do_git}"
+   then
+      status_stage "==> Commiting Dev Mode Changes"
+      commit_dev_mode "${sdir}" || return 1
+      
+      status_stage "==> Confirming Git Changes"
+      confirm_git_push_changes "${sdir}"
+      
+      status_stage "==> Pushing to Git"
+      git_push_ref "$1" || return 1
+   fi
+   
    return 0
 }
 

build-support/scripts/publish.sh

@@ -16,6 +16,13 @@ function usage {
 cat <<-EOF
 Usage: ${SCRIPT_NAME}  [<options ...>]
 
+Description:
+
+   This script will "publish" a Consul release. It expects a prebuilt release in 
+   pkg/dist matching the version in the repo and a clean git status. It will 
+   prompt you to confirm the consul version and git changes you are going to 
+   publish prior to pushing to git and to releases.hashicorp.com.
+
 Options:                       
    -s | --source     DIR         Path to source to build.
                                  Defaults to "${SOURCE_DIR}"

build-support/scripts/release.sh

@@ -16,11 +16,28 @@ function usage {
 cat <<-EOF
 Usage: ${SCRIPT_NAME}  [<options ...>]
 
+Description:
+   
+   This script will do a full release build of Consul. Building each component
+   is done within a docker container. In addition to building Consul this
+   script will do a few more things.
+   
+      * Update version/version*.go files
+      * Update CHANGELOG.md to put things into release mode
+      * Create a release commit. It changes in the commit include the CHANGELOG.md
+        version files and the assetfs.
+      * Tag the release
+      * Generate the SHA256SUMS file for the binaries
+      * Sign the SHA256SUMS file with a GPG key
+
+
 Options:                       
    -s | --source     DIR         Path to source to build.
                                  Defaults to "${SOURCE_DIR}"
                                  
-   -t | --tag        BOOL        Whether to add a release commit and tag the build
+   -t | --tag        BOOL        Whether to add a release commit and tag the build. 
+                                 This also controls whether we put the tree into
+                                 release mode
                                  Defaults to 1.
                                  
    -b | --build      BOOL        Whether to perform the build of the ui's, assetfs and

build-support/scripts/version.sh

@@ -16,6 +16,11 @@ function usage {
 cat <<-EOF
 Usage: ${SCRIPT_NAME}  [<options ...>]
 
+Description:
+
+   This script is just a convenience around discover what the Consul
+   version would be if you were to build it. 
+
 Options:                       
    -s | --source     DIR         Path to source to build.
                                  Defaults to "${SOURCE_DIR}"