Commit 74eebc3d authored by Greg Althaus's avatar Greg Althaus
Browse files

Merge branch 'update-build-process-for-docs' into 'v4' 

build: doc update

See merge request !913
parents b35dfcdb ac7d0be5

conf.py

+73 −65
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ import os 
import requests

import shutil

import threading

import subprocess



# If extensions (or modules to document with autodoc) are in another directory,

# add these directories to sys.path here. If the directory is relative to the
@@ -329,6 +330,9 @@ texinfo_documents = [ 
# If true, do not generate a @detailmenu in the "Top" node's menu.

#texinfo_no_detailmenu = False



# Get branch

branch = subprocess.check_output(['tools/version-docs.sh']).decode('utf-8').strip()



# Skip download

skip = False

if os.path.isfile('skip-doc-download'):
@@ -427,6 +431,7 @@ class FileDownloader(): 


    def get_file(self, link, filename, path, session=None):

        """ Downloads the file"""

        link = link.format(branch)

        thread = threading.Thread(target=self.t_getfile, args=(link, filename, path, session))

        thread.start()
@@ -450,70 +455,73 @@ def fetch_n_save(urls, path): 
        td.get_file(link, filename, path, None)



def fetch_urls(url):

    url = url.format(branch)

    r = requests.get(url, verify=True, stream=True)

    r.raw.decode_content = True

    with open("/tmp/tmp.builder", 'wb') as f:

        shutil.copyfileobj(r.raw, f)

    with open('/tmp/tmp.builder') as my_file:

        testsite_array = my_file.readlines()

    if testsite_array[0].startswith("<?xml"):

        print("Error: " + url)

    return testsite_array





urls = [

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/ad-auth.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/agent.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/basic-store.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/bios.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/burnin.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/callback.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/certs.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/chef-bootstrap.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/classify.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/cloud-wrappers.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/cohesity.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/coreos.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/dell-support.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/dev-library.rst",

    "https://rebar-catalog.s3.us-west-2.amazonaws.com/docs/docker-context.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/drp-community-content.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/drp-community-contrib.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/drp-prom-mon.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/edge-lab.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/eikon.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/endpoint-exec.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/event2audit.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/filebeat.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/flash.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/flexiflow.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/honeycomb.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/hpe-support.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/image-builder.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/image-deploy.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/ipmi.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/kube-lib.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/krib.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/kvm-test.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/lenovo-support.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/os-other.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/packer-builder.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/packet-ipmi.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/proxmox.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rack.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/raid.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rancheros.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/slack.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/sledgehammer-builder.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/solidfire.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/task-library.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/tenant-controller.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/tower.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/ux-views.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/universal.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/validation.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/vault.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/virtualbox-ipmi.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/vmware.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/vmware-lib.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/ad-auth.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/agent.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/basic-store.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/bios.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/burnin.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/callback.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/certs.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/chef-bootstrap.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/classify.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/cloud-wrappers.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/cohesity.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/coreos.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/dell-support.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/dev-library.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/docker-context.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/drp-community-content.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/drp-community-contrib.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/drp-prom-mon.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/edge-lab.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/eikon.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/endpoint-exec.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/event2audit.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/filebeat.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/flash.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/flexiflow.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/honeycomb.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/hpe-support.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/image-builder.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/image-deploy.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/ipmi.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/kube-lib.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/krib.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/kvm-test.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/lenovo-support.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/os-other.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/packer-builder.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/packet-ipmi.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/proxmox.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rack.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/raid.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rancheros.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/slack.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/sledgehammer-builder.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/solidfire.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/task-library.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/tenant-controller.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/tower.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/ux-views.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/universal.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/validation.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/vault.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/virtualbox-ipmi.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/vmware.rst",

    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/vmware-lib.rst",

]

fetch_n_save(urls, path="doc/content-packages")
@@ -528,26 +536,26 @@ urls = [ 
]

fetch_n_save(urls, path="doc/BankN")



urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rel-notes/drp.filelist")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rel-notes/drp.filelist")

fetch_n_save(urls, path="doc/rel_notes/drp")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rel-notes/drp-server.filelist")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rel-notes/drp-server.filelist")

fetch_n_save(urls, path="doc/rel_notes/drp-server")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rel-notes/drp-content.filelist")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rel-notes/drp-content.filelist")

fetch_n_save(urls, path="doc/rel_notes/drp-content")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rel-notes/drp-plugins.filelist")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rel-notes/drp-plugins.filelist")

fetch_n_save(urls, path="doc/rel_notes/drp-plugins")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rel-notes/rackn-plugins.filelist")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rel-notes/rackn-plugins.filelist")

fetch_n_save(urls, path="doc/rel_notes/rackn-plugins")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rel-notes/rackn-cohesity.filelist")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rel-notes/rackn-cohesity.filelist")

fetch_n_save(urls, path="doc/rel_notes/rackn-cohesity")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rel-notes/rackn-solidfire.filelist")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rel-notes/rackn-solidfire.filelist")

fetch_n_save(urls, path="doc/rel_notes/rackn-solidfire")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/rel-notes/rackn-ux.filelist")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/rel-notes/rackn-ux.filelist")

fetch_n_save(urls, path="doc/rel_notes/rackn-ux")



fetch_n_save_as("https://gitlab.com/rackn/labs/-/raw/v4/README.rst", path="doc", filename="labs.rst")

fetch_n_save_as("https://gitlab.com/rackn/labs/-/raw/v4/images/lab_urls_param.png", path="doc/images", filename="lab_urls_param.png")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/labs/labs.filelist")

fetch_n_save_as("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/README.rst", path="doc", filename="labs.rst")

fetch_n_save_as("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/images/lab_urls_param.png", path="doc/images", filename="lab_urls_param.png")

urls = fetch_urls("https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/{0}/labs/labs.filelist")

fetch_n_save(urls, path="doc/labs")



dest_folder = 'doc/content-packages'

tools/build_rel_notes.sh

+8 −6
Original line number Diff line number Diff line
@@ -6,6 +6,8 @@ rm -rf rel_notes 
mkdir -p rel_notes



BR=$(git rev-parse --abbrev-ref HEAD)

. tools/version.sh

branch=${MajorV}.${MinorV}



PREV=""

git tag | sort -V | while read rev
@@ -51,15 +53,15 @@ do 
        rm rel_notes/$rev.txt

done



mkdir -p rebar-catalog/docs/rel-notes/$subdir

cp rel_notes/* rebar-catalog/docs/rel-notes/$subdir

mkdir -p rebar-catalog/docs/$branch/rel-notes/$subdir

cp rel_notes/* rebar-catalog/docs/$branch/rel-notes/$subdir



aws s3 ls rebar-catalog/docs/rel-notes/$subdir/ | awk '{ print $4 }' > files.current

ls rebar-catalog/docs/rel-notes/$subdir >> files.current

aws s3 ls rebar-catalog/docs/$branch/rel-notes/$subdir/ | awk '{ print $4 }' > files.current

ls rebar-catalog/docs/$branch/rel-notes/$subdir >> files.current

sort -r -V files.current >> new-files.current

cat new-files.current | while read f

do

    echo "https://rebar-catalog.s3-us-west-2.amazonaws.com/docs/rel-notes/$subdir/$f"

    echo "https://rebar-catalog.s3-us-west-2.amazonaws.com/docs/$branch/rel-notes/$subdir/$f"

done > new-files2.current

cp new-files2.current rebar-catalog/docs/rel-notes/$subdir.filelist

cp new-files2.current rebar-catalog/docs/$branch/rel-notes/$subdir.filelist

rm -f new-files.current new-files2.current files.current

tools/version-docs.sh

0 → 100755
+4 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash



. tools/version.sh 2>/dev/null >/dev/null

echo ${MajorV}.${MinorV}