Skip to content
Snippets Groups Projects
Commit a5056b99 authored by Megan Filo's avatar Megan Filo :exclamation:
Browse files

Update sitemap creation logic

parent fbaab16c
No related branches found
No related tags found
1 merge request!132184Update sitemap creation logic
......@@ -139,11 +139,13 @@ variables:
if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAMESPACE_ID != "6543"'
 
# "always" - master branch in the original repo, OR any merge request
.if-master-original-repo-or-merge-request: &if-master-original-repo-or-merge-request
.if-master-original-repo-or-merge-request:
&if-master-original-repo-or-merge-request
if: '($CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_ID == "7764") || $CI_MERGE_REQUEST_IID'
 
# if triggered by the scheduled job to cleanup old deleted files, only on master branch in the original repo
.if-schedule-deploy-cleanup-old-deleted-files-master-original-repo: &if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
.if-schedule-deploy-cleanup-old-deleted-files-master-original-repo:
&if-schedule-deploy-cleanup-old-deleted-files-master-original-repo
if: '$CI_PIPELINE_SOURCE == "schedule" && $RUN_SCHEDULED_DEPLOY_CLEANUP_OLD_DELETED_FILES == "true" && $CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_ID == "7764"'
 
### COMMON JOBS REUSED VIA `extends`:
......@@ -451,8 +453,8 @@ packaged-assets-webpack-build-and-prod-deploy:
.assets-build-base:
extends: .build-base
variables:
DESTINATION_PATH_REGEXES: '^ico/'
MIDDLEMAN_CONFIG_FILE_NAME: 'config_assets.rb'
DESTINATION_PATH_REGEXES: "^ico/"
MIDDLEMAN_CONFIG_FILE_NAME: "config_assets.rb"
 
.assets-build-script: &assets-build-script
- bundle exec middleman build --bail
......@@ -514,7 +516,7 @@ handbook-build-and-review-deploy:
# NOTE: this `.upload-artifacts` is only needed by `check-handbook-edit-links`
- .upload-artifacts
rules:
- <<: *if-merge-request-canonical-pipeline
- <<: *if-merge-request-canonical-pipeline
script:
- echo "Handbook has been migrated to the new handbook site."
 
......@@ -531,10 +533,10 @@ handbook-build-and-prod-deploy:
 
.uncategorized-build-base:
extends: .build-base
parallel: 5
parallel: 3
 
.uncategorized-build-script: &uncategorized-build-script
- if [[ "$CI_NODE_INDEX" == "4" ]]; then bin/crop-team-pictures; fi
- if [[ "$CI_NODE_INDEX" == "2" ]]; then bin/crop-team-pictures; fi
- cd sites/uncategorized
- find source/images/team -type f ! -name '*-crop.jpg' -delete
- bundle exec middleman build --bail
......@@ -586,8 +588,8 @@ uncategorized-build-and-prod-deploy:
# 15 minute default timeout. So, it is increased to 30m to be safe.
timeout: 30m
variables:
INCLUDE_GENERATORS: 'true'
CI_BUILD_PROXY_RESOURCE: 'true'
INCLUDE_GENERATORS: "true"
CI_BUILD_PROXY_RESOURCE: "true"
cache:
key: "build_proxy_resource_ruby-3.0-bullseye"
policy: pull-push
......@@ -595,7 +597,8 @@ uncategorized-build-and-prod-deploy:
- tmp/cache
- vendor
 
.uncategorized-proxy-resource-build-script: &uncategorized-proxy-resource-build-script
.uncategorized-proxy-resource-build-script:
&uncategorized-proxy-resource-build-script
- cd sites/uncategorized && bundle exec middleman build --bail && cd -
 
uncategorized-proxy-resource-build-only:
......@@ -721,7 +724,7 @@ deploy-redirects-for-mr:
- .review-environment
environment:
on_stop: review-stop 2/2
needs: ['validate-redirects']
needs: ["validate-redirects"]
rules:
- <<: *if-merge-request-canonical-pipeline
changes:
......@@ -732,7 +735,7 @@ deploy-redirects-for-mr:
.deploy-site-router:
stage: build-and-deploy
extends: .redirects-prepare
needs: ['validate-redirects']
needs: ["validate-redirects"]
rules:
- <<: *if-master-original-repo
changes:
......@@ -880,8 +883,8 @@ include:
# Can be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/388988
- template: Jobs/Secret-Detection.latest.gitlab-ci.yml
- template: Jobs/Dependency-Scanning.latest.gitlab-ci.yml
- project: 'gitlab-org/frontend/untamper-my-lockfile'
file: 'templates/merge_request_pipelines.yml'
- project: "gitlab-org/frontend/untamper-my-lockfile"
file: "templates/merge_request_pipelines.yml"
 
# Override https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
code_quality:
......@@ -903,16 +906,15 @@ gemnasium-dependency_scanning:
when: never
- <<: *if-master-original-repo-or-merge-request # specific
exists:
- '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'
- '{composer.lock,*/composer.lock,*/*/composer.lock}'
- '{gems.locked,*/gems.locked,*/*/gems.locked}'
- '{go.sum,*/go.sum,*/*/go.sum}'
- '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}'
- '{package-lock.json,*/package-lock.json,*/*/package-lock.json}'
- '{yarn.lock,*/yarn.lock,*/*/yarn.lock}'
- '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}'
- '{conan.lock,*/conan.lock,*/*/conan.lock}'
- "{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}"
- "{composer.lock,*/composer.lock,*/*/composer.lock}"
- "{gems.locked,*/gems.locked,*/*/gems.locked}"
- "{go.sum,*/go.sum,*/*/go.sum}"
- "{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}"
- "{package-lock.json,*/package-lock.json,*/*/package-lock.json}"
- "{yarn.lock,*/yarn.lock,*/*/yarn.lock}"
- "{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}"
- "{conan.lock,*/conan.lock,*/*/conan.lock}"
 
# == END Dependency Scanning jobs
 
......@@ -936,13 +938,13 @@ secret_detection:
fi
check-handbook-edit-links:
extends: .bundle-install
# NOTE: Because of the time it would takes to download all artifacts, and because the non-handbook portions of the site may change in the future,
# we are only checking links in the handbook instead of all build jobs, even though there are edit links on non-handbook pages
# which are still shared for now. So, this only provides coverage for major breakages or regressions in the edit
# links logic for pages in the handbook, not the rest of the site.
#
# Note that the source for the `lib/lint/check_handbook_edit_links.rb` could still be used for checking the rest of the site
# (if the proper skip regexes were added), but this job is only running it against the files generated by the handbook build job.
# NOTE: Because of the time it would takes to download all artifacts, and because the non-handbook portions of the site may change in the future,
# we are only checking links in the handbook instead of all build jobs, even though there are edit links on non-handbook pages
# which are still shared for now. So, this only provides coverage for major breakages or regressions in the edit
# links logic for pages in the handbook, not the rest of the site.
#
# Note that the source for the `lib/lint/check_handbook_edit_links.rb` could still be used for checking the rest of the site
# (if the proper skip regexes were added), but this job is only running it against the files generated by the handbook build job.
stage: test
rules:
- <<: *if-merge-request-canonical-pipeline
......@@ -951,10 +953,10 @@ check-handbook-edit-links:
# the list of changes to trigger it are restricted to files currently involved in rendering the edit links.
- data/monorepo.yml
- helpers/custom_helpers.rb
- '**/edit_page.html.haml'
- '**/footer.html.haml'
- '**/handbook-page-toc.haml'
- '**/check_handbook_edit_links.rb'
- "**/edit_page.html.haml"
- "**/footer.html.haml"
- "**/handbook-page-toc.haml"
- "**/check_handbook_edit_links.rb"
needs:
- handbook-build-and-review-deploy
script:
......@@ -991,7 +993,7 @@ review-stop 1/2:
when: manual
allow_failure: true
variables:
DEPLOY_DELETE_APP: 'true'
DEPLOY_DELETE_APP: "true"
environment:
action: stop
script:
......@@ -1015,7 +1017,7 @@ deploy-cleanup-old-deleted-files:
- packaged-assets-webpack-build-only
- assets-build-only
variables:
DEPLOY_CLEANUP_OLD_DELETED_FILES: 'true'
DEPLOY_CLEANUP_OLD_DELETED_FILES: "true"
script:
- scripts/pull_blog_artifacts.rb
- scripts/pull_buyer_experience_artifacts.rb
......@@ -1024,7 +1026,7 @@ deploy-cleanup-old-deleted-files:
danger-review:
image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
rules:
- if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID'
- if: "$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID"
stage: test
needs: []
script:
......
......@@ -2,17 +2,14 @@
 
require 'middleman'
 
#
# This middleman extension is used to speed up builds on CI
#
# We effectively do this by leveraging our CI parallelism features and split the build into
# distinct partitions. Note that this is actively being refactored as part of the monorepo effort.
# distinct partitions. Note that this is actively being refactored as part of the monorepo effort.
# See docs/monorepo.md for more details:
#
# * `PROXY_RESOURCE`: We leverage the `proxy` feature of middleman to generate e.g. job pages,
# direction pages and others
# * `BLOG_POST_OLD`: All blog posts up to 2017
# * `BLOG_POST_NEW`: All blog posts since 2018
# * `RELEASE_BLOG`: Release blog posts
# * `COMPANY`: All company pages and team images
# * `ALL_OTHERS`: All pages and files which do not fit into the categories above
......@@ -21,8 +18,6 @@ require 'middleman'
# parallelism off, it will simply be a noop extension
class PartialBuildUncategorized < Middleman::Extension
PROXY_RESOURCE = "proxy resources"
BLOG_POST_OLD = "blog posts old (up to 2017)"
BLOG_POST_NEW = "blog posts new (since 2018)"
RELEASE_BLOG = "release blog"
COMPANY = "Company pages and team images"
ALL_OTHERS = "all other pages"
......@@ -44,20 +39,16 @@ class PartialBuildUncategorized < Middleman::Extension
 
return unless @enabled
 
expected_jobs = 5
expected_jobs = 3
 
raise "#{self.class.name}: If you want to enable parallel builds, please use exactly #{expected_jobs} parallel jobs. (CI_NODE_TOTAL='#{ENV['CI_NODE_TOTAL']}')" unless ENV['CI_NODE_TOTAL'].to_i == expected_jobs
 
@partial = case ENV['CI_NODE_INDEX']
when "1"
BLOG_POST_OLD
when "2"
BLOG_POST_NEW
when "3"
RELEASE_BLOG
when "4"
when "2"
COMPANY
when "5"
when "3"
ALL_OTHERS
else
raise "#{self.class.name}: Invalid Build Partial #{ENV['CI_NODE_INDEX']}. At the moment we only support 1 to #{expected_jobs}"
......@@ -117,30 +108,6 @@ class PartialBuildUncategorized < Middleman::Extension
 
# PARTIAL BUILD RESOURCES
 
def blog_post_old?(resource)
return true if resource.destination_path =~ %r{sitemaps/sitemap_blog_page_old} # Child sitemap for this partial build
!proxy_resource?(resource) &&
resource.destination_path.start_with?(
'blog/2011',
'blog/2012',
'blog/2012',
'blog/2013',
'blog/2014',
'blog/2015',
'blog/2016',
'blog/2017'
)
end
def blog_post_new?(resource)
return true if resource.destination_path =~ %r{sitemaps/sitemap_blog_page_new} # Child sitemap for this partial build
!proxy_resource?(resource) &&
((resource.destination_path.start_with?('blog/') && !blog_post_old?(resource)) ||
resource.destination_path.end_with?('atom.xml'))
end
def release_blog?(resource)
return true if resource.destination_path =~ %r{sitemaps/sitemap_release_blog} # Child sitemap for this partial build
 
......@@ -180,8 +147,6 @@ class PartialBuildUncategorized < Middleman::Extension
return false if independently_built_resource?(resource)
 
!proxy_resource?(resource) &&
!blog_post_new?(resource) &&
!blog_post_old?(resource) &&
!release_blog?(resource) &&
!company?(resource)
end
......@@ -192,10 +157,6 @@ class PartialBuildUncategorized < Middleman::Extension
case @partial
when PROXY_RESOURCE
proxy_resource?(resource)
when BLOG_POST_OLD
blog_post_old?(resource)
when BLOG_POST_NEW
blog_post_new?(resource)
when RELEASE_BLOG
release_blog?(resource)
when COMPANY
......
......@@ -9,9 +9,9 @@ xml.sitemapindex 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9' do
end
<% end %>
 
<%# Add handbook site child sitemap to sitemap index %>
<%# Add new blog sitemap to sitemap index %>
xml.sitemap do
xml.loc 'https://about.gitlab.com/handbook/sitemap_handbook.xml'
xml.loc 'https://about.gitlab.com/blog/sitemap.xml'
end
 
<%# Add buyer experience sitemap to sitemap index %>
......
<%= File.read(File.expand_path('source/includes/sitemap_template.xml.builder', monorepo_root)) %>
\ No newline at end of file
<%= File.read(File.expand_path('source/includes/sitemap_template.xml.builder', monorepo_root)) %>
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment