Skip to content
Snippets Groups Projects
Commit 89e36bd1 authored by Miguel Angel Reina Ortega's avatar Miguel Angel Reina Ortega
Browse files

Add functionality to upgrade index.html in all projects

parent 5ea48eaa
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 8
− 89
View file @ 89e36bd1
...@@ -6,20 +6,13 @@ ...@@ -6,20 +6,13 @@
workflow: workflow:
rules: rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_COMMIT_BRANCH
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
when: never
variables: variables:
stages: stages:
- build - build
- generation - upgrade
- publication
- web
Build pythonForPandocFilter docker image: Build pythonForPandocFilter docker image:
stage: build stage: build
...@@ -35,85 +28,11 @@ Build pythonForPandocFilter docker image: ...@@ -35,85 +28,11 @@ Build pythonForPandocFilter docker image:
- generateChangemarks/changemarks.py - generateChangemarks/changemarks.py
- generateChangemarks/addTrackedChanges.py - generateChangemarks/addTrackedChanges.py
Word CR text: Upgrade index for pages:
stage: generation stage: upgrade
only:
- merge_requests
before_script:
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh
- chmod +x generate_changemarks.sh
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
script:
- echo 'CR contribution'
- mkdir docs
- ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx
artifacts:
paths:
- docs/
expose_as: 'Word CR text'
Protect branch:
stage: generation
when: on_success
needs: ["Word CR text"]
only:
- merge_requests
script:
- |
curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30"
Publish spec:
stage: publication
only:
- tags
before_script: before_script:
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh
- chmod +x publish_spec.sh
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
- |
export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
script: script:
- echo 'Publishing spec' - ./upgrade_pages.sh ${ACCESS_TOKEN}
- ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME
artifacts:
name: "${SPEC_NAME}_${CI_COMMIT_TAG}"
paths:
- "*_${CI_COMMIT_TAG}.docx"
pages:
stage: web
when: on_success
rules: rules:
- if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG - changes:
variables: - index.html
TAG_NAME: $CI_COMMIT_TAG \ No newline at end of file
- if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"
variables:
TAG_NAME: $CLEAN_WEB_PAGES
before_script:
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh
- chmod +x publish_on_pages.sh
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html
- |
export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
script:
- echo 'Publishing on pages'
- ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME
artifacts:
paths:
- public
\ No newline at end of file
generateChangemarks/.gitlab-ci.yml 0 → 100644
+ 109
− 0
View file @ 89e36bd1
# CI/CD:
#
# GENERATION triggered by:
# - merge request
#
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event" # Creation of Merge Request (generation of CR)
- if: $CI_COMMIT_TAG # Creation of tag (publish spec)
- if: $CI_PIPELINE_SOURCE == "web" # Removal of a specific entry
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
when: never # Pipeline to be run manually as defined in the contribution procedure as it is not known when the CR is ready
- if: $CI_PIPELINE_SOURCE == "trigger" # Upgrade of the index.html page
variables:
stages:
- generation
- publication
- web
Word CR:
stage: generation
only:
- merge_requests
before_script:
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh
- chmod +x generate_changemarks.sh
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
script:
- echo 'CR contribution'
- mkdir docs
- ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx
artifacts:
paths:
- docs/
expose_as: 'Word CR'
Protect branch:
stage: generation
when: on_success
needs: ["Word CR"]
only:
- merge_requests
script:
- |
curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30"
Publish spec:
stage: publication
only:
- tags
before_script:
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh
- chmod +x publish_spec.sh
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
- |
export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
script:
- echo 'Publishing spec'
- ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME
artifacts:
name: "${SPEC_NAME}_${CI_COMMIT_TAG}"
paths:
- "*_${CI_COMMIT_TAG}.docx"
pages:
stage: web
when: on_success
rules:
- if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG
variables:
TAG_NAME: $CI_COMMIT_TAG
- if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"
variables:
TAG_NAME: $CLEAN_WEB_PAGES
- if: $CI_PIPELINE_SOURCE == "trigger"
variables:
TAG_NAME: "upgrade"
before_script:
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh
- chmod +x publish_on_pages.sh
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py
- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html
- |
export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
- |
sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' index.html
script:
- echo 'Publishing on pages'
- ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME;
artifacts:
paths:
- public
\ No newline at end of file
publish_on_pages.sh
+ 7
− 2
View file @ 89e36bd1
...@@ -5,7 +5,7 @@ DOCKER_IMAGE=python:3.9-slim-bullseye ...@@ -5,7 +5,7 @@ DOCKER_IMAGE=python:3.9-slim-bullseye
echo "\n------ Checking for docker image --------" echo "\n------ Checking for docker image --------"
docker pull "$DOCKER_IMAGE" docker pull "$DOCKER_IMAGE"
echo "Pages URL:" $2 echo "Pages URL:" $2
echo "PROJECT NAME:" $3 echo "PROJECT NAME:" $3 # Not used anymore, consider to remove it
echo "TAG NAME:" $4 echo "TAG NAME:" $4
echo "------ Get the previous GitLab Pages content --------" echo "------ Get the previous GitLab Pages content --------"
...@@ -15,7 +15,10 @@ if [ -e "content.zip" ]; then unzip "content.zip"; rm "content.zip"; fi ...@@ -15,7 +15,10 @@ if [ -e "content.zip" ]; then unzip "content.zip"; rm "content.zip"; fi
set -e set -e
echo "------ Add/update content --------" echo "------ Add/update content --------"
if [ $6 == 'true' ]; then if [ $4 == 'upgrade' ]; then
echo 'Upgrading index.html...'
docker run --rm -v $(pwd):/tmp -w /tmp "$DOCKER_IMAGE" pip install bs4 && python3 $1 -u "$7_$4" "$4/$7_$4.docx" "$5"
elif [ $6 == 'true' ]; then
echo 'Removing all web pages content...' echo 'Removing all web pages content...'
rm -r public/*; rm -r public/*;
elif [ $6 == 'false' ]; then elif [ $6 == 'false' ]; then
...@@ -41,3 +44,5 @@ exit 0 ...@@ -41,3 +44,5 @@ exit 0
#$CI_PROJECT_NAME #$CI_PROJECT_NAME
#$CI_COMMIT_REF_NAME #$CI_COMMIT_REF_NAME
#index.html #index.html
#CLEAN_WEB_PAGES
#SPEC_NAME
updateIndex.py
+ 53
− 1
View file @ 89e36bd1
...@@ -9,6 +9,53 @@ ...@@ -9,6 +9,53 @@
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import argparse, os import argparse, os
def upgradeIndex(args: argparse.Namespace) -> None:
# Check if index.htm exists
if not os.path.exists('public/index.html'):
print("'index.html' does not exist yet, Upgrade is not needed")
exit()
else:
# Read the source HTML file
with open('public/index.html', 'r', encoding='utf-8') as source_file:
source_html = source_file.read()
# Read the destination HTML file
with open(args.indexTemplate, 'r', encoding='utf-8') as dest_file:
dest_html = dest_file.read()
print(source_html)
print(dest_html)
# Parse both HTML files
source_soup = BeautifulSoup(source_html, 'html.parser')
dest_soup = BeautifulSoup(dest_html, 'html.parser')
print(source_soup)
print(dest_soup)
# Find the <ul> element in both the source and destination HTML
source_ul = source_soup.find('ul', id='document-list')
dest_ul = dest_soup.find('ul', id='document-list')
print(source_ul)
print(dest_ul)
# Copy the contents of the source <ul> to the destination <ul>
if dest_ul and source_ul:
dest_ul.contents.extend(source_ul.contents)
else:
print("ERROR: List not found in source and destination index files...")
exit(1)
# Save the updated destination HTML
with open('public/index.html', 'w', encoding='utf-8') as dest_updated_file:
dest_updated_file.write(str(dest_soup))
# Read index HTML
with open('public/index.html', 'r', encoding='utf-8') as file:
html_content = file.read()
print("Index.html upgraded")
def updateIndex(args:argparse.Namespace) -> None: def updateIndex(args:argparse.Namespace) -> None:
# Check if index.htm exists # Check if index.htm exists
...@@ -63,10 +110,15 @@ if __name__ == '__main__': ...@@ -63,10 +110,15 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-re', '--removeEntry', action='store_true', required=False, default=False, parser.add_argument('-re', '--removeEntry', action='store_true', required=False, default=False,
help="Indicate to remove the given entry") help="Indicate to remove the given entry")
parser.add_argument('-u', '--upgrade', action='store_true', required=False, default=False,
help="Upgrade the index.html")
parser.add_argument('documentName', help = 'Document name to add to the list') parser.add_argument('documentName', help = 'Document name to add to the list')
parser.add_argument('documentLink', help = 'Document location to add to the list') parser.add_argument('documentLink', help = 'Document location to add to the list')
parser.add_argument('indexTemplate', help = 'Index template document') parser.add_argument('indexTemplate', help = 'Index template document')
args = parser.parse_args() args = parser.parse_args()
if args.upgrade:
upgradeIndex(args)
else:
updateIndex(args) updateIndex(args)
upgrade_pages.sh 0 → 100755
+ 30
− 0
View file @ 89e36bd1
#!/bin/bash
GITLAB_HOST="http://git.onem2m.org"
SPECIFICATIONS_GROUP="2501"
NS_GITLAB_HOST=$(echo $GITLAB_HOST | cut -d'/' -f3)
echo "\n------ Getting all projects IDs --------"
# Get a list of projects in the group using GitLab API
#PROJECT_IDS=$(curl --header "PRIVATE-TOKEN: $1" "$GITLAB_HOST/api/v4/groups/$SPECIFICATIONS_GROUP/projects" | grep -oP '"id":(?!'$SPECIFICATIONS_GROUP')\d+' | cut -d':' -f2)
PROJECT_IDS=$(curl "$GITLAB_HOST/api/v4/groups/$SPECIFICATIONS_GROUP/projects" | grep -oP '"id":(?!'$SPECIFICATIONS_GROUP')\d+' | cut -d':' -f2)
echo $PROJECT_IDS
#PROJECT_IDS="41" # Use TS-0019 for debugging
# Trigger pipeline for each project. Triggered pipeline pages job will handle the upgrade of the index.html
for PROJECT_ID in $PROJECT_IDS; do
echo "------ Preparing specification project --------"
#WORKAROUND UNTIL ACCESS TOKEN WORKS FOR TRIGGERING A PIPELINE (right now, it is allowed but it seems NOT to work)
#create a pipeline trigger token first
echo "------ Creating pipeline trigger token --------"
PIPELINE_TRIGGER_TOKEN_ID=$(curl --request POST --header "PRIVATE-TOKEN:$1" --form description="Temporary pipeline trigger token for tools/scripts" https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers | grep -o '"id":[^,]*,"token"'|cut -d',' -f1 | cut -d':' -f2)
PIPELINE_TRIGGER_TOKEN=$(curl --header "PRIVATE-TOKEN:$1" https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers/$PIPELINE_TRIGGER_TOKEN_ID | grep -o '"token":[^,]*'|cut -d'"' -f4)
echo "------ Triggering pipeline --------"
#trigger the pipeline
curl -X POST --fail -F token=$PIPELINE_TRIGGER_TOKEN -F ref=main https://git.onem2m.org/api/v4/projects/$PROJECT_ID/trigger/pipeline
echo "------ Deleting pipeline trigger token (NOTE that information about the token that caused the pipeline will NOT be available for the pipeline) --------"
#remove the pipeline trigger token
curl --request DELETE --header "PRIVATE-TOKEN:$1" "https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers/$PIPELINE_TRIGGER_TOKEN_ID"
done
exit 0
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment