update-doc.sh

#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

function exit_with_usage {
  cat << EOF

  USAGE: update-doc.sh [spark] [flink]

  This script generates the documentation for the individual Bahir (Spark) and
  Bahir-Flink modules from the contents of the respective module's README.md
  files as found in the Bahir and Bahir-Flink source repositories.

     bahir (source repo)       bahir-website
     .                         └─site
     .                           └─docs
     .                             └─spark
     ├─sql-streaming-mqtt            └─current
     │ └─README.md          ─────>     ├─spark-sql-streaming-mqtt.md
     ├─streaming-akka                  │
     │ └─README.md          ─────>     ├─spark-streaming-akka.md
     ├─streaming-mqtt                  │
     │ └─README.md          ─────>     ├─spark-streaming-mqtt.md
     ├─streaming-twitter               │
     │ └─README.md          ─────>     ├─spark-streaming-twitter.md
     └─streaming-zeromq                │
       └─README.md          ─────>     └─spark-streaming-zeromq.md

     bahir-flink (source repo)   bahir-website
     .                           └─site
     .                             └─docs
     .                               └─flink
     ├─sql-streaming-mqtt              └─current
     │ └─README.md          ─────>       ├─flink-streaming-activemq.md
     ├─streaming-akka                    │
     │ └─README.md          ─────>       ├─flink-streaming-akka.md
     ├─streaming-mqtt                    │
     │ └─README.md          ─────>       ├─flink-streaming-flume.md
     ├─streaming-twitter                 │
     │ └─README.md          ─────>       ├─flink-streaming-netty.md
     └─streaming-zeromq                  │
       └─README.md          ─────>       └─flink-streaming-redis.md


  Page header with license text comes from the respective template files
  under:
    site/docs/spark/templates
    site/docs/flink/templates

EOF
  exit 1
}

if [[ "$@" =~ "-h" ]]; then
    exit_with_usage
fi

REPOS="${@:-'spark flink'}"

set -e

BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

SPARK_WEBSITE_TEMPLATES_DIR=site/docs/spark/templates
SPARK_WEBSITE_DOC_DIR=site/docs/spark/current
SPARK_REPO_NAME=bahir
SPARK_BAHIR_SOURCE_DIR=target/$SPARK_REPO_NAME

FLINK_WEBSITE_TEMPLATES_DIR=site/docs/flink/templates
FLINK_WEBSITE_DOC_DIR=site/docs/flink/current
FLINK_REPO_NAME=bahir-flink
FLINK_BAHIR_SOURCE_DIR=target/$FLINK_REPO_NAME

GIT_REF=${GIT_REF:-master}

function checkout_code {
    # Checkout code
    cd "$BASE_DIR" # make sure we're in the base dir

    rm -rf target
    mkdir target
    cd target
    git clone "https://gitbox.apache.org/repos/asf/$1.git" --quiet
    cd "$1"
    git checkout "$GIT_REF"
    git_hash=$(git rev-parse --short HEAD)
    echo "Checked out $1 git hash $git_hash"

    git clean -d -f -x

    cd "$BASE_DIR" # return to base dir
}

function update_docs {
    WEBSITE_TEMPLATES_DIR=$1
    WEBSITE_DOC_DIR=$2
    SOURCE_DIR=$3
    shift 3

    [ ! -d "$WEBSITE_DOC_DIR" ] && mkdir "$WEBSITE_DOC_DIR"

    echo "Updating documents ..."

    while [ $# -ne 0 ]; do
        echo "  $WEBSITE_DOC_DIR/$1.md   <--   $SOURCE_DIR/$2/README.md"
        rm -f "$WEBSITE_DOC_DIR/$1.md"
        cp    "$WEBSITE_TEMPLATES_DIR/$1.template" "$WEBSITE_DOC_DIR/$1.md"
        cat   "$SOURCE_DIR/$2/README.md"        >> "$WEBSITE_DOC_DIR/$1.md"
        shift 2
    done
}

function check_version_strings {
    if grep -q -r "[0-9]-SNAPSHOT" "$1"/*.md ; then
        echo
        echo "TODO: Replace '...-SNAPSHOT' version strings:"
        echo
        grep -r -n "[0-9]-SNAPSHOT" "$1"/*.md | sed 's/^/  /' | grep --color "[0-9.]*-SNAPSHOT"
        echo
        echo "i.e. to replace '2.1.0-SNAPSHOT' with '2.0.2' run the following command:"
        echo
        echo "  perl -i -pe 's/2.1.0-SNAPSHOT/2.0.2/g' $1/*.md"
        echo
    else
        echo
        echo "Generated files:"
        echo
        ls "$1"/*.md | xargs -n1 | sed -e 's|'"$(pwd -P)"/'||g'
    fi
}

function update_spark {
    echo
    echo "================= Updating Apache Spark Extension documents ================="
    echo

    checkout_code $SPARK_REPO_NAME

    update_docs "$SPARK_WEBSITE_TEMPLATES_DIR"              \
        "$SPARK_WEBSITE_DOC_DIR"  "$SPARK_BAHIR_SOURCE_DIR" \
        spark-sql-cloudant        sql-cloudant              \
        spark-sql-streaming-akka  sql-streaming-akka        \
        spark-sql-streaming-mqtt  sql-streaming-mqtt        \
        spark-streaming-akka      streaming-akka            \
        spark-streaming-mqtt      streaming-mqtt            \
        spark-streaming-pubsub    streaming-pubsub          \
        spark-streaming-pubnub    streaming-pubnub          \
        spark-streaming-twitter   streaming-twitter         \
        spark-streaming-zeromq    streaming-zeromq

    check_version_strings "$SPARK_WEBSITE_DOC_DIR"
}

function update_flink {
    echo
    echo "================= Updating Apache Flink Extension documents ================="
    echo

    checkout_code $FLINK_REPO_NAME

    update_docs "$FLINK_WEBSITE_TEMPLATES_DIR"              \
        "$FLINK_WEBSITE_DOC_DIR"  "$FLINK_BAHIR_SOURCE_DIR" \
        flink-streaming-activemq  flink-connector-activemq  \
        flink-streaming-akka      flink-connector-akka      \
        flink-streaming-flume     flink-connector-flume     \
        flink-streaming-influxdb  flink-connector-influxdb  \
        flink-streaming-influxdb2  flink-connector-influxdb2  \
        flink-streaming-kudu      flink-connector-kudu      \
        flink-streaming-netty     flink-connector-netty     \
        flink-streaming-pinot     flink-connector-pinot     \
        flink-streaming-redis     flink-connector-redis

    check_version_strings "$FLINK_WEBSITE_DOC_DIR"
}


[[ "$REPOS" =~ "spark" ]] && update_spark
[[ "$REPOS" =~ "flink" ]] && update_flink

set +e