diff --git a/.drone.env b/.drone.env deleted file mode 100644 index 2cd4b1b7284..00000000000 --- a/.drone.env +++ /dev/null @@ -1,3 +0,0 @@ -# The test runner source for UI tests -WEB_COMMITID=6165b4e47a47e3a216107e96f90825190f10632e -WEB_BRANCH=stable-12.3 diff --git a/.drone.star b/.drone.star deleted file mode 100644 index 35bba97746b..00000000000 --- a/.drone.star +++ /dev/null @@ -1,4091 +0,0 @@ -"""oCIS CI definition -""" - -# Production release tags -# NOTE: need to be updated if new production releases are determined -# - follow semver -# - omit 'v' prefix -PRODUCTION_RELEASE_TAGS = ["5.0", "7", "8"] - -# images -ALPINE_GIT = "alpine/git:latest" -APACHE_TIKA = "apache/tika:3.2.2.0-full" -CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1" -COLLABORA_CODE = "collabora/code:24.04.5.1.1" -AXLLENT_MAILPIT = "axllent/mailpit:v1.22.3" -MINIO_MC = "minio/mc:RELEASE.2021-10-07T04-19-58Z" -OC_CI_ALPINE = "owncloudci/alpine:latest" -OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier:latest" -OC_CI_CLAMAVD = "owncloudci/clamavd" -OC_CI_DRONE_ANSIBLE = "owncloudci/drone-ansible:latest" -OC_CI_DRONE_SKIP_PIPELINE = "owncloudci/drone-skip-pipeline" -OC_CI_GOLANG = "owncloudci/golang:1.25" -OC_CI_NODEJS = "owncloudci/nodejs:%s" -OC_CI_PHP = "owncloudci/php:%s" -OC_CI_WAIT_FOR = "owncloudci/wait-for:latest" -OC_CS3_API_VALIDATOR = "owncloud/cs3api-validator:0.2.1" -OC_LITMUS = "owncloudci/litmus:latest" -OC_UBUNTU = "owncloud/ubuntu:20.04" -ONLYOFFICE_DOCUMENT_SERVER = "onlyoffice/documentserver:9.0.0" -PLUGINS_CODACY = "plugins/codacy:1" -PLUGINS_DOCKER = "plugins/docker:latest" -PLUGINS_GH_PAGES = "plugins/gh-pages:1" -PLUGINS_GITHUB_RELEASE = "plugins/github-release:1" -PLUGINS_GIT_ACTION = "plugins/git-action:1" -PLUGINS_MANIFEST = "plugins/manifest:1" - -# 1.4.0 is required due to minio compatibility -PLUGINS_S3 = "plugins/s3:1.4.0" -PLUGINS_S3_CACHE = "plugins/s3-cache:1" -REDIS = "redis:6-alpine" -SONARSOURCE_SONAR_SCANNER_CLI = "sonarsource/sonar-scanner-cli:11.0" -KEYCLOAK_IMAGE = "quay.io/keycloak/keycloak:26.2.5" -POSTGRES_ALPINE_IMAGE = "postgres:alpine3.18" -TRIVY_IMAGE = "aquasec/trivy:latest" -K3D_IMAGE = "ghcr.io/k3d-io/k3d:5-dind" -CODACY_COVERAGE_REPORTER = "codacy/codacy-coverage-reporter:latest" -CS3ORG_WOPISERVER = "cs3org/wopiserver:v10.4.0" -OC_CI_WOPI_VALIDATOR = "owncloudci/wopi-validator" - -# the hugo version needs to be the same as in owncloud.github.io -OC_CI_HUGO_STATIC_IMAGE = "hugomods/hugo:base-0.129.0" - -DEFAULT_PHP_VERSION = "8.4" -DEFAULT_NODEJS_VERSION = "24" - -dirs = { - "base": "/drone/src", - "web": "/drone/src/webTestRunner", - "zip": "/drone/src/zip", - "webZip": "/drone/src/zip/web.tar.gz", - "webPnpmZip": "/drone/src/zip/pnpm-store.tar.gz", - "gobinTar": "go-bin.tar.gz", - "gobinTarPath": "/drone/src/go-bin.tar.gz", - "ocisConfig": "tests/config/drone/ocis-config.json", - "ocis": "/srv/app/tmp/ocis", - "ocisRevaDataRoot": "/srv/app/tmp/ocis/owncloud/data", - "ocisWrapper": "/drone/src/tests/ociswrapper", - "bannedPasswordList": "tests/config/drone/banned-password-list.txt", - "ocmProviders": "tests/config/drone/providers.json", -} - -# OCIS URLs -OCIS_SERVER_NAME = "ocis-server" -OCIS_URL = "https://%s:9200" % OCIS_SERVER_NAME -OCIS_DOMAIN = "%s:9200" % OCIS_SERVER_NAME -FED_OCIS_SERVER_NAME = "federation-%s" % OCIS_SERVER_NAME -OCIS_FED_URL = "https://%s:10200" % FED_OCIS_SERVER_NAME -OCIS_FED_DOMAIN = "%s:10200" % FED_OCIS_SERVER_NAME - -# urls for k8s deployment -OCIS_URL_K8S = "https://%s" % OCIS_SERVER_NAME -OCIS_FED_URL_K8S = "https://%s" % FED_OCIS_SERVER_NAME - -# emails -EMAIL_PORT = "8025" -EMAIL_SMTP_PORT = "1025" -EMAIL_SMTP_HOST = "email" -EMAIL_SMTP_SENDER = "ownCloud " - -S3_CACHE_SERVER = "https://cache.owncloud.com" -S3_CACHE_BUCKET = "cache" -S3_PUBLIC_CACHE_BUCKET = "public" - -# configuration -config = { - "cs3ApiTests": { - "skip": False, - }, - "wopiValidatorTests": { - "skip": False, - }, - "k6LoadTests": { - "skip": False, - }, - "localApiTests": { - "contractAndLock": { - "suites": [ - "apiContract", - "apiLocks", - ], - "skip": False, - "k8s": True, - }, - "settingsAndNotification": { - "suites": [ - "apiSettings", - "apiNotification", - "apiCors", - ], - "skip": False, - "withRemotePhp": [False], - "k8s": True, - "emailNeeded": True, - "extraEnvironment": { - "EMAIL_HOST": EMAIL_SMTP_HOST, - "EMAIL_PORT": EMAIL_PORT, - }, - "extraServerEnvironment": { - "OCIS_ADD_RUN_SERVICES": "notifications", - "NOTIFICATIONS_SMTP_HOST": EMAIL_SMTP_HOST, - "NOTIFICATIONS_SMTP_PORT": EMAIL_SMTP_PORT, - "NOTIFICATIONS_SMTP_INSECURE": "true", - "NOTIFICATIONS_SMTP_SENDER": EMAIL_SMTP_SENDER, - "NOTIFICATIONS_DEBUG_ADDR": "0.0.0.0:9174", - }, - }, - "graphUser": { - "suites": [ - "apiGraphUser", - ], - "skip": False, - "withRemotePhp": [False], - "k8s": True, - }, - "spaces": { - "suites": [ - "apiSpaces", - ], - "skip": False, - "k8s": True, - }, - "spacesShares": { - "suites": [ - "apiSpacesShares", - ], - "skip": False, - "k8s": True, - }, - "davOperations": { - "suites": [ - "apiSpacesDavOperation", - "apiDownloads", - "apiAsyncUpload", - "apiDepthInfinity", - "apiArchiver", - "apiActivities", - ], - "skip": False, - "k8s": True, - }, - "groupAndSearch1": { - "suites": [ - "apiSearch1", - "apiGraph", - "apiGraphGroup", - ], - "skip": False, - "k8s": True, - }, - "search2": { - "suites": [ - "apiSearch2", - "apiSearchContent", - ], - "tikaNeeded": True, - "skip": False, - "k8s": True, - }, - "sharingNg1": { - "suites": [ - "apiSharingNgShares", - "apiReshare", - "apiSharingNgPermissions", - ], - "skip": False, - "withRemotePhp": [False], - "k8s": True, - }, - "sharingNgAdditionalShareRole": { - "suites": [ - "apiSharingNgAdditionalShareRole", - ], - "skip": False, - "k8s": True, - "withRemotePhp": [False], - }, - "sharingNgShareInvitation": { - "suites": [ - "apiSharingNgDriveInvitation", - "apiSharingNgItemInvitation", - ], - "skip": False, - "withRemotePhp": [False], - "k8s": True, - }, - "sharingNgLinkShare": { - "suites": [ - "apiSharingNgDriveLinkShare", - "apiSharingNgItemLinkShare", - "apiSharingNgLinkShareManagement", - ], - "skip": False, - "k8s": True, - "withRemotePhp": [False], - }, - "antivirus": { - "suites": [ - "apiAntivirus", - ], - "skip": False, - "k8s": True, - "antivirusNeeded": True, - "extraServerEnvironment": { - "ANTIVIRUS_SCANNER_TYPE": "clamav", - "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310", - "POSTPROCESSING_STEPS": "virusscan", - "OCIS_ADD_RUN_SERVICES": "antivirus", - "ANTIVIRUS_DEBUG_ADDR": "0.0.0.0:9277", - }, - }, - "ocm": { - "suites": [ - "apiOcm", - ], - "skip": False, - "k8s": True, - "withRemotePhp": [False], - "federationServer": True, - "emailNeeded": True, - "extraEnvironment": { - "EMAIL_HOST": EMAIL_SMTP_HOST, - "EMAIL_PORT": EMAIL_PORT, - }, - "extraServerEnvironment": { - "OCIS_ADD_RUN_SERVICES": "ocm,notifications", - "OCIS_ENABLE_OCM": True, - "OCM_OCM_INVITE_MANAGER_INSECURE": True, - "OCM_OCM_SHARE_PROVIDER_INSECURE": True, - "OCM_OCM_STORAGE_PROVIDER_INSECURE": True, - "OCM_OCM_PROVIDER_AUTHORIZER_PROVIDERS_FILE": "%s" % dirs["ocmProviders"], - # mail notifications - "NOTIFICATIONS_SMTP_HOST": EMAIL_SMTP_HOST, - "NOTIFICATIONS_SMTP_PORT": EMAIL_SMTP_PORT, - "NOTIFICATIONS_SMTP_INSECURE": "true", - "NOTIFICATIONS_SMTP_SENDER": EMAIL_SMTP_SENDER, - }, - }, - "authApp": { - "suites": [ - "apiAuthApp", - ], - "skip": False, - "k8s": True, - "withRemotePhp": [False], - "extraServerEnvironment": { - "OCIS_ADD_RUN_SERVICES": "auth-app", - "PROXY_ENABLE_APP_AUTH": True, - }, - }, - "wopi": { - "suites": [ - "apiCollaboration", - ], - "skip": False, - "k8s": True, - "withRemotePhp": [False], - "collaborationServiceNeeded": True, - "extraServerEnvironment": { - "GATEWAY_GRPC_ADDR": "0.0.0.0:9142", - }, - }, - "cliCommands": { - "suites": [ - "cliCommands", - "apiServiceAvailability", - ], - "skip": False, - "withRemotePhp": [False], - "antivirusNeeded": True, - "emailNeeded": True, - "extraEnvironment": { - "EMAIL_HOST": EMAIL_SMTP_HOST, - "EMAIL_PORT": EMAIL_PORT, - }, - "extraServerEnvironment": { - "NOTIFICATIONS_SMTP_HOST": EMAIL_SMTP_HOST, - "NOTIFICATIONS_SMTP_PORT": EMAIL_SMTP_PORT, - "NOTIFICATIONS_SMTP_INSECURE": "true", - "NOTIFICATIONS_SMTP_SENDER": EMAIL_SMTP_SENDER, - "NOTIFICATIONS_DEBUG_ADDR": "0.0.0.0:9174", - "ANTIVIRUS_SCANNER_TYPE": "clamav", - "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310", - "ANTIVIRUS_DEBUG_ADDR": "0.0.0.0:9277", - "OCIS_ADD_RUN_SERVICES": "antivirus,notifications", - }, - }, - }, - "coreApiTests": { - "1": { - "suites": [ - "coreApiAuth", - "coreApiCapabilities", - "coreApiFavorites", - "coreApiMain", - "coreApiVersions", - ], - "skip": False, - "withRemotePhp": [False], - "k8s": True, - }, - "2": { - "suites": [ - "coreApiShareManagementBasicToShares", - "coreApiShareManagementToShares", - ], - "skip": False, - "k8s": True, - "withRemotePhp": [False], - }, - "3": { - "suites": [ - "coreApiSharees", - "coreApiSharePublicLink2", - ], - "skip": False, - "withRemotePhp": [False], - "k8s": True, - }, - "4": { - "suites": [ - "coreApiShareOperationsToShares1", - "coreApiShareOperationsToShares2", - "coreApiSharePublicLink1", - "coreApiShareCreateSpecialToShares1", - "coreApiShareCreateSpecialToShares2", - "coreApiShareUpdateToShares", - ], - "skip": False, - "k8s": True, - "withRemotePhp": [False], - }, - "5": { - "suites": [ - "coreApiTrashbin", - "coreApiTrashbinRestore", - "coreApiWebdavEtagPropagation1", - "coreApiWebdavEtagPropagation2", - ], - "skip": False, - "k8s": True, - }, - "6": { - "suites": [ - "coreApiWebdavDelete", - "coreApiWebdavOperations", - "coreApiWebdavMove2", - ], - "skip": False, - "k8s": True, - }, - "7": { - "suites": [ - "coreApiWebdavProperties", - ], - "skip": False, - "k8s": True, - }, - "8": { - "suites": [ - "coreApiWebdavMove1", - "coreApiWebdavPreviews", - "coreApiWebdavUpload", - "coreApiWebdavUploadTUS", - ], - "skip": False, - "k8s": True, - }, - }, - "e2eTests": { - "part": { - "skip": False, - "totalParts": 4, # divide and run all suites in parts (divide pipelines) - "xsuites": ["search", "app-provider", "ocm", "keycloak"], # suites to skip - }, - "search": { - "skip": False, - "suites": ["search"], # suites to run - "tikaNeeded": True, - }, - "keycloak": { - "skip": False, - "suites": ["journeys", "keycloak"], - "keycloakNeeded": True, - }, - }, - "e2eMultiService": { - "testSuites": { - "skip": False, - "suites": [ - "smoke", - "shares", - "search", - "journeys", - "file-action", - "spaces", - ], - "tikaNeeded": True, - }, - }, - "binaryReleases": { - "os": ["linux", "darwin"], - }, - "dockerReleases": { - "architectures": ["arm64", "amd64"], - }, - "litmus": True, - "codestyle": True, -} - -# volume for steps to cache Go dependencies between steps of a pipeline -# GOPATH must be set to /go inside the image, which is the case -stepVolumeGo = \ - { - "name": "gopath", - "path": "/go", - } - -# volume for pipeline to cache Go dependencies between steps of a pipeline -# to be used in combination with stepVolumeGo -pipelineVolumeGo = \ - { - "name": "gopath", - "temp": {}, - } - -stepVolumeOcisStorage = \ - { - "name": "storage", - "path": "/root/.ocis", - } - -# minio mc environment variables -MINIO_MC_ENV = { - "CACHE_BUCKET": S3_CACHE_BUCKET, - "MC_HOST": S3_CACHE_SERVER, - "AWS_ACCESS_KEY_ID": { - "from_secret": "cache_s3_access_key", - }, - "AWS_SECRET_ACCESS_KEY": { - "from_secret": "cache_s3_secret_key", - }, -} - -DRONE_HTTP_PROXY_ENV = { - "HTTP_PROXY": { - "from_secret": "drone_http_proxy", - }, - "HTTPS_PROXY": { - "from_secret": "drone_http_proxy", - }, -} - -def pipelineDependsOn(pipeline, dependant_pipelines): - if "depends_on" in pipeline.keys(): - pipeline["depends_on"] = pipeline["depends_on"] + getPipelineNames(dependant_pipelines) - else: - pipeline["depends_on"] = getPipelineNames(dependant_pipelines) - return pipeline - -def pipelinesDependsOn(pipelines, dependant_pipelines): - pipes = [] - for pipeline in pipelines: - pipes.append(pipelineDependsOn(pipeline, dependant_pipelines)) - - return pipes - -def getPipelineNames(pipelines = []): - """getPipelineNames returns names of pipelines as a string array - - Args: - pipelines: array of drone pipelines - - Returns: - names of the given pipelines as string array - """ - names = [] - for pipeline in pipelines: - names.append(pipeline["name"]) - return names - -def main(ctx): - """main is the entrypoint for drone - - Args: - ctx: drone passes a context with information which the pipeline can be adapted to - - Returns: - none - """ - - pipelines = [] - - build_release_helpers = \ - changelog() + \ - documentation() + \ - licenseCheck(ctx) - - test_pipelines = \ - codestyle(ctx) + \ - checkGherkinLint(ctx) + \ - checkTestSuitesInExpectedFailures(ctx) + \ - buildWebCache(ctx) + \ - getGoBinForTesting(ctx) + \ - buildOcisBinaryForTesting(ctx) + \ - checkStarlark() + \ - build_release_helpers + \ - testOcisAndUploadResults(ctx) + \ - testPipelines(ctx) - - build_release_pipelines = \ - dockerReleases(ctx) + \ - binaryReleases(ctx) - - test_pipelines.append( - pipelineDependsOn( - purgeBuildArtifactCache(ctx), - testPipelines(ctx), - ), - ) - - test_pipelines.append( - pipelineDependsOn( - uploadAPITestCoverageReport(ctx), - testPipelines(ctx), - ), - ) - - pipelines = test_pipelines + build_release_pipelines - - # nightly Trivy security scan (non-blocking) - pipelines.append(trivyScan(ctx)) - - if ctx.build.event == "cron": - pipelines = \ - pipelines + \ - example_deploys(ctx) - else: - pipelines = \ - pipelines + \ - pipelinesDependsOn( - example_deploys(ctx), - pipelines, - ) - - pipelineSanityChecks(ctx, pipelines) - return pipelines - -def cachePipeline(name, steps): - return { - "kind": "pipeline", - "type": "docker", - "name": "build-%s-cache" % name, - "clone": { - "disable": True, - }, - "steps": steps, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/**", - "refs/pull/**", - ], - }, - } - -def buildWebCache(ctx): - return [ - cachePipeline("web", generateWebCache(ctx)), - cachePipeline("web-pnpm", generateWebPnpmCache(ctx)), - ] - -def testOcisAndUploadResults(ctx): - pipeline = testOcis(ctx) - - ###################################################################### - # The triggers have been disabled for now, since the govulncheck can # - # not silence single, acceptable vulnerabilities. # - # See https://github.com/owncloud/ocis/issues/9527 for more details. # - # FIXME: RE-ENABLE THIS ASAP!!! # - ###################################################################### - - scan_result_upload = uploadScanResults(ctx) - scan_result_upload["depends_on"] = getPipelineNames([pipeline]) - - #security_scan = scanOcis(ctx) - #return [security_scan, pipeline, scan_result_upload] - return [pipeline, scan_result_upload] - -def testPipelines(ctx): - pipelines = [] - - if config["litmus"]: - pipelines += litmus(ctx, "ocis") - - if "skip" not in config["cs3ApiTests"] or not config["cs3ApiTests"]["skip"]: - pipelines.append(cs3ApiTests(ctx, "ocis")) - if "skip" not in config["wopiValidatorTests"] or not config["wopiValidatorTests"]["skip"]: - pipelines.append(wopiValidatorTests(ctx, "ocis", "builtin")) - pipelines.append(wopiValidatorTests(ctx, "ocis", "cs3")) - - pipelines += localApiTestPipeline(ctx) - pipelines += coreApiTestPipeline(ctx) - - pipelines += e2eTestPipeline(ctx) + multiServiceE2ePipeline(ctx) - - if ("skip" not in config["k6LoadTests"] or not config["k6LoadTests"]["skip"]) and ("k6-test" in ctx.build.title.lower() or ctx.build.event == "cron"): - pipelines += k6LoadTests(ctx) - - return pipelines - -def getGoBinForTesting(ctx): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "get-go-bin-cache", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "unit-tests") + - checkGoBinCache() + - cacheGoBin(), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - }] - -def checkGoBinCache(): - return [{ - "name": "check-go-bin-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "bash -x %s/tests/config/drone/check_go_bin_cache.sh %s %s" % (dirs["base"], dirs["base"], dirs["gobinTar"]), - ], - }] - -def cacheGoBin(): - return [ - { - "name": "bingo-get", - "image": OC_CI_GOLANG, - "commands": [ - "make bingo-update", - ], - "volumes": [stepVolumeGo], - "environment": DRONE_HTTP_PROXY_ENV, - }, - { - "name": "archive-go-bin", - "image": OC_UBUNTU, - "commands": [ - "tar -czvf %s /go/bin" % dirs["gobinTarPath"], - ], - "volumes": [stepVolumeGo], - }, - { - "name": "cache-go-bin", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - # .bingo folder will change after 'bingo-get' - # so get the stored hash of a .bingo folder - "BINGO_HASH=$(cat %s/.bingo_hash)" % dirs["base"], - # cache using the minio client to the public bucket (long term bucket) - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r %s s3/$CACHE_BUCKET/ocis/go-bin/$BINGO_HASH" % (dirs["gobinTarPath"]), - ], - "volumes": [stepVolumeGo], - }, - ] - -def restoreGoBinCache(): - return [ - { - "name": "restore-go-bin-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "BINGO_HASH=$(cat %s/.bingo/* | sha256sum | cut -d ' ' -f 1)" % dirs["base"], - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a s3/$CACHE_BUCKET/ocis/go-bin/$BINGO_HASH/%s %s" % (dirs["gobinTar"], dirs["base"]), - ], - "volumes": [stepVolumeGo], - }, - { - "name": "extract-go-bin-cache", - "image": OC_UBUNTU, - "commands": [ - "tar -xvmf %s -C /" % dirs["gobinTarPath"], - ], - "volumes": [stepVolumeGo], - }, - ] - -def testOcis(ctx): - steps = skipIfUnchanged(ctx, "unit-tests") + restoreGoBinCache() + makeGoGenerate("") + [ - { - "name": "golangci-lint", - "image": OC_CI_GOLANG, - "commands": [ - "mkdir -p cache/checkstyle", - "make ci-golangci-lint", - "mv checkstyle.xml cache/checkstyle/checkstyle.xml", - ], - "environment": DRONE_HTTP_PROXY_ENV, - "volumes": [stepVolumeGo], - }, - { - "name": "test", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "mkdir -p cache/coverage", - "make test", - "mv coverage.out cache/coverage/", - ], - "volumes": [stepVolumeGo], - }, - { - "name": "scan-result-cache", - "image": PLUGINS_S3, - "settings": { - "endpoint": MINIO_MC_ENV["MC_HOST"], - "bucket": MINIO_MC_ENV["CACHE_BUCKET"], - "source": "cache/**/*", - "target": "%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - "path_style": True, - "access_key": MINIO_MC_ENV["AWS_ACCESS_KEY_ID"], - "secret_key": MINIO_MC_ENV["AWS_SECRET_ACCESS_KEY"], - }, - }, - ] - - return { - "kind": "pipeline", - "type": "docker", - "name": "linting_and_unitTests", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": steps, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - "depends_on": getPipelineNames(getGoBinForTesting(ctx)), - "volumes": [pipelineVolumeGo], - } - -def scanOcis(ctx): - steps = skipIfUnchanged(ctx, "unit-tests") + restoreGoBinCache() + makeGoGenerate("") + [ - { - "name": "govulncheck", - "image": OC_CI_GOLANG, - "commands": [ - "make govulncheck", - ], - "environment": DRONE_HTTP_PROXY_ENV, - "volumes": [stepVolumeGo], - }, - ] - - return { - "kind": "pipeline", - "type": "docker", - "name": "go-vulnerability-scanning", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": steps, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - "depends_on": getPipelineNames(getGoBinForTesting(ctx)), - "volumes": [pipelineVolumeGo], - } - -def buildOcisBinaryForTesting(ctx): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "build_ocis_binary_for_testing", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + - makeNodeGenerate("") + - makeGoGenerate("") + - build() + - buildDebug() + - rebuildBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin"), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - }] - -def uploadScanResults(ctx): - sonar_env = { - "SONAR_TOKEN": { - "from_secret": "sonar_token", - }, - } - if ctx.build.event == "pull_request": - sonar_env.update({ - "SONAR_PULL_REQUEST_BASE": "%s" % (ctx.build.target), - "SONAR_PULL_REQUEST_BRANCH": "%s" % (ctx.build.source), - "SONAR_PULL_REQUEST_KEY": "%s" % (ctx.build.ref.replace("refs/pull/", "").split("/")[0]), - }) - - fork_handling = [] - if ctx.build.source_repo != "" and ctx.build.source_repo != ctx.repo.slug: - fork_handling = [ - "git remote add fork https://github.com/%s.git" % (ctx.build.source_repo), - "git fetch fork", - ] - - return { - "kind": "pipeline", - "type": "docker", - "name": "upload-scan-results", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "clone": { - "disable": True, # Sonarcloud does not apply issues on already merged branch - }, - "steps": [ - { - "name": "clone", - "image": ALPINE_GIT, - "commands": [ - # Always use the owncloud/ocis repository as base to have an up to date default branch. - # This is needed for the skipIfUnchanged step, since it references a commit on master (which could be absent on a fork) - "git clone https://github.com/%s.git ." % (ctx.repo.slug), - ] + fork_handling + - [ - "git checkout $DRONE_COMMIT", - ], - }, - ] + skipIfUnchanged(ctx, "unit-tests") + [ - { - "name": "sync-from-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "mkdir -p cache", - "mc alias set cachebucket $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc mirror cachebucket/$CACHE_BUCKET/%s/%s/cache cache/ || true" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - ], - }, - { - "name": "codacy", - "image": CODACY_COVERAGE_REPORTER, - "environment": { - "CODACY_PROJECT_TOKEN": { - "from_secret": "codacy_token", - }, - }, - "commands": [ - "/app/codacy-coverage-reporter report --force-coverage-parser go -r cache/coverage/coverage.out", - ], - }, - { - "name": "sonarcloud", - "image": SONARSOURCE_SONAR_SCANNER_CLI, - "environment": sonar_env, - }, - { - "name": "purge-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "mc alias set cachebucket $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc rm --recursive --force cachebucket/$CACHE_BUCKET/%s/%s/cache || true" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - ], - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - "status": [ - "success", - "failure", - ], - }, - } - -def vendorbinCodestyle(phpVersion): - return [{ - "name": "vendorbin-codestyle", - "image": OC_CI_PHP % phpVersion, - "environment": { - "COMPOSER_HOME": "%s/.cache/composer" % dirs["base"], - }, - "commands": [ - "make vendor-bin-codestyle", - ], - }] - -def vendorbinCodesniffer(phpVersion): - return [{ - "name": "vendorbin-codesniffer", - "image": OC_CI_PHP % phpVersion, - "environment": { - "COMPOSER_HOME": "%s/.cache/composer" % dirs["base"], - }, - "commands": [ - "make vendor-bin-codesniffer", - ], - }] - -def checkTestSuitesInExpectedFailures(ctx): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "check-suites-in-expected-failures", - "steps": [ - { - "name": "check-suites", - "image": OC_CI_ALPINE, - "commands": [ - "%s/tests/acceptance/check-deleted-suites-in-expected-failure.sh" % dirs["base"], - ], - }, - ], - "trigger": { - "ref": [ - "refs/pull/**", - ], - }, - }] - -def checkGherkinLint(ctx): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "check-gherkin-standard", - "steps": [ - { - "name": "lint-feature-files", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "npm install -g @gherlint/gherlint@1.1.0", - "make test-gherkin-lint", - ], - }, - ], - "trigger": { - "ref": [ - "refs/pull/**", - ], - }, - }] - -def codestyle(ctx): - pipelines = [] - - if "codestyle" not in config: - return [] - - default = { - "phpVersions": [DEFAULT_PHP_VERSION], - } - - if "defaults" in config: - if "codestyle" in config["defaults"]: - for item in config["defaults"]["codestyle"]: - default[item] = config["defaults"]["codestyle"][item] - - codestyleConfig = config["codestyle"] - - if type(codestyleConfig) == "bool": - if codestyleConfig: - # the config has 'codestyle' true, so specify an empty dict that will get the defaults - codestyleConfig = {} - else: - return pipelines - - if len(codestyleConfig) == 0: - # 'codestyle' is an empty dict, so specify a single section that will get the defaults - codestyleConfig = {"doDefault": {}} - - for category, matrix in codestyleConfig.items(): - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - for phpVersion in params["phpVersions"]: - name = "coding-standard-php%s" % phpVersion - - result = { - "kind": "pipeline", - "type": "docker", - "name": name, - "workspace": { - "base": "/drone", - "path": "src", - }, - "steps": skipIfUnchanged(ctx, "lint") + - vendorbinCodestyle(phpVersion) + - vendorbinCodesniffer(phpVersion) + - [ - { - "name": "php-style", - "image": OC_CI_PHP % phpVersion, - "commands": [ - "make test-php-style", - ], - }, - { - "name": "check-env-var-annotations", - "image": OC_CI_PHP % phpVersion, - "commands": [ - "make check-env-var-annotations", - ], - }, - ], - "depends_on": [], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - } - - pipelines.append(result) - - return pipelines - -def localApiTestPipeline(ctx): - pipelines = [] - - with_remote_php = [False] - if ctx.build.event == "cron" or "full-ci" in ctx.build.title.lower(): - with_remote_php.append(True) - - defaults = { - "suites": {}, - "skip": False, - "extraEnvironment": {}, - "extraServerEnvironment": {}, - "storages": ["ocis"], - "emailNeeded": False, - "antivirusNeeded": False, - "tikaNeeded": False, - "federationServer": False, - "collaborationServiceNeeded": False, - "extraCollaborationEnvironment": {}, - "withRemotePhp": with_remote_php, - "k8s": False, - } - - if "localApiTests" in config: - for name, matrix in config["localApiTests"].items(): - if "skip" not in matrix or not matrix["skip"]: - params = {} - for item in defaults: - params[item] = matrix[item] if item in matrix else defaults[item] - for storage in params["storages"]: - for run_with_remote_php in params["withRemotePhp"]: - run_on_k8s = params["k8s"] and ctx.build.event == "cron" - - #################### - # SETUP STEPS # - #################### - setup_steps = skipIfUnchanged(ctx, "acceptance-tests") - if params["tikaNeeded"]: - setup_steps += waitForServices("tika", ["tika:9998"]) - if params["antivirusNeeded"]: - setup_steps += waitForServices("clamav", ["clamav:3310"]) - if params["emailNeeded"]: - setup_steps += waitForServices("email", ["%s:%s" % (EMAIL_SMTP_HOST, EMAIL_PORT)]) - if params["collaborationServiceNeeded"]: - setup_steps += waitForServices("online-offices", ["collabora:9980", "onlyoffice:443", "fakeoffice:8080"]) - - if run_on_k8s: - enable_auth_app = False - if "OCIS_ADD_RUN_SERVICES" in params["extraServerEnvironment"]: - enable_auth_app = "auth-app" in params["extraServerEnvironment"]["OCIS_ADD_RUN_SERVICES"] - setup_steps += ocisServerK8s( - params["federationServer"], - params["antivirusNeeded"], - params["emailNeeded"], - params["tikaNeeded"], - params["collaborationServiceNeeded"], - enable_auth_app, - ) - - if params["collaborationServiceNeeded"]: - setup_steps += exposeNodePortsK8s([ - ["collaboration-collabora", 9300], - ["collaboration-fakeoffice", 9300], - ["collaboration-onlyoffice", 9300], - ["collaboration-fakeoffice", 9304], - ]) - else: - setup_steps += restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") - setup_steps += ocisServer( - storage, - extra_server_environment = params["extraServerEnvironment"], - with_wrapper = True, - tika_enabled = params["tikaNeeded"], - volumes = [stepVolumeOcisStorage], - ) - - if params["federationServer"]: - setup_steps += ocisServer( - storage, - deploy_type = "federation", - extra_server_environment = params["extraServerEnvironment"], - ) - if params["collaborationServiceNeeded"]: - setup_steps += wopiCollaborationService("fakeoffice") + \ - wopiCollaborationService("collabora") + \ - wopiCollaborationService("onlyoffice") + \ - ocisHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) - - #################### - # SERVICES # - #################### - services = [] - if params["tikaNeeded"]: - services += tikaService() - if params["emailNeeded"]: - services += emailService() - if params["antivirusNeeded"]: - services += clamavService() - if params["collaborationServiceNeeded"]: - services += fakeOffice() + collaboraService() + onlyofficeService() - if run_on_k8s: - services += k3sCluster(ocm = params["federationServer"]) - if params["federationServer"]: - services += k3sCluster(name = FED_OCIS_SERVER_NAME, ocm = True) - - #################### - # PIPELINE # - #################### - pipeline_name = "%s-%s" % ("CLI" if name.startswith("cli") else "API", name) - if run_with_remote_php: - pipeline_name += "-withRemotePhp" - if run_on_k8s: - pipeline_name += "-k8s" - pipeline = { - "kind": "pipeline", - "type": "docker", - "name": pipeline_name, - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": setup_steps + - localApiTests( - name, - params["suites"], - storage, - params["extraEnvironment"], - run_with_remote_php, - k8s = run_on_k8s, - ) + - apiTestFailureLog() + - (generateCoverageFromAPITest(ctx, name) if not run_on_k8s else []), - "services": services, - "depends_on": [] if run_on_k8s else getPipelineNames(buildOcisBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - "volumes": [{ - "name": "storage", - "temp": {}, - }], - } - pipelines.append(pipeline) - return pipelines - -def generateCoverageFromAPITest(ctx, name): - environment = { - "GOCOVERDIR": "reports", - } - - return [ - { - "name": "coverageReport-%s" % name, - "image": OC_CI_GOLANG, - "environment": environment, - "commands": [ - "go tool covdata percent -i=$GOCOVERDIR -o=coverage-%s.out" % name, - ], - }, - { - "name": "coverage-locate", - "image": OC_UBUNTU, - "commands": [ - "mkdir -p cache/acceptance/coverage/", - "mv coverage-%s.out cache/acceptance/coverage/" % name, - ], - }, - { - "name": "coverage-cache-1", - "image": PLUGINS_S3, - "settings": { - "endpoint": MINIO_MC_ENV["MC_HOST"], - "bucket": MINIO_MC_ENV["CACHE_BUCKET"], - "source": "cache/acceptance/coverage/coverage-%s.out" % name, - "target": "%s/%s/coverage" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - "path_style": True, - "access_key": MINIO_MC_ENV["AWS_ACCESS_KEY_ID"], - "secret_key": MINIO_MC_ENV["AWS_SECRET_ACCESS_KEY"], - }, - }, - ] - -def localApiTests(name, suites, storage = "ocis", extra_environment = {}, with_remote_php = False, k8s = False): - test_dir = "%s/tests/acceptance" % dirs["base"] - expected_failures_file = "%s/expected-failures-localAPI-on-%s-storage.md" % (test_dir, storage.upper()) - wrapper_url = "http://%s:5200" % OCIS_SERVER_NAME - ocis_url = OCIS_URL - ocis_fed_url = OCIS_FED_URL - - if k8s: - ocis_url = OCIS_URL_K8S - ocis_fed_url = OCIS_FED_URL_K8S - wrapper_url = "http://ociswrapper:5200" - - environment = { - "TEST_SERVER_URL": ocis_url, - "TEST_SERVER_FED_URL": ocis_fed_url, - "OCIS_REVA_DATA_ROOT": "%s" % (dirs["ocisRevaDataRoot"] if storage == "owncloud" else ""), - "STORAGE_DRIVER": storage, - "BEHAT_SUITES": ",".join(suites), - "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnGraph&&~@skipOnOcis-%s-Storage" % ("OC" if storage == "owncloud" else "OCIS"), - "EXPECTED_FAILURES_FILE": expected_failures_file, - "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, - "OCIS_WRAPPER_URL": wrapper_url, - "WITH_REMOTE_PHP": with_remote_php, - "COLLABORATION_SERVICE_URL": "http://ocis-server:9304" if k8s else "http://wopi-fakeoffice:9300", - "K8S": k8s, - } - - for item in extra_environment: - environment[item] = extra_environment[item] - - return [{ - "name": "localApiTests-%s" % name, - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "environment": environment, - "commands": [ - # merge the expected failures - "" if with_remote_php else "cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file), - "mkdir -p /etc/ocis/", - "make -C %s test-acceptance-api" % (dirs["base"]), - ], - "volumes": [stepVolumeOcisStorage], - }] - -def cs3ApiTests(ctx, storage): - return { - "kind": "pipeline", - "type": "docker", - "name": "cs3ApiTests", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + - restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + - ocisServer(storage, [], [], "cs3api_validator") + - [ - { - "name": "cs3ApiTests", - "image": OC_CS3_API_VALIDATOR, - "environment": {}, - "commands": [ - "/usr/bin/cs3api-validator /var/lib/cs3api-validator --endpoint=%s:9142" % OCIS_SERVER_NAME, - ], - }, - ], - "depends_on": getPipelineNames(buildOcisBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - } - -def wopiValidatorTests(ctx, storage, wopiServerType): - testgroups = [ - "BaseWopiViewing", - "CheckFileInfoSchema", - "EditFlows", - "Locks", - "AccessTokens", - "GetLock", - "ExtendedLockLength", - "FileVersion", - "Features", - ] - builtinOnlyTestGroups = [ - "PutRelativeFile", - "RenameFileIfCreateChildFileIsNotSupported", - ] - - validatorTests = [] - wopiServer = [] - extra_server_environment = {} - - if wopiServerType == "cs3": - wopiServer = [ - { - "name": "wopi-fakeoffice", - "image": CS3ORG_WOPISERVER, - "detach": True, - "commands": [ - "cp %s/tests/config/drone/wopiserver.conf /etc/wopi/wopiserver.conf" % (dirs["base"]), - "echo 123 > /etc/wopi/wopisecret", - "/app/wopiserver.py", - ], - }, - ] - else: - extra_server_environment = { - "OCIS_EXCLUDE_RUN_SERVICES": "app-provider", - } - - wopiServer = wopiCollaborationService("fakeoffice") - - for testgroup in testgroups: - validatorTests.append({ - "name": "wopiValidatorTests-%s" % testgroup, - "image": OC_CI_WOPI_VALIDATOR, - "commands": [ - "export WOPI_TOKEN=$(cat accesstoken)", - "echo $WOPI_TOKEN", - "export WOPI_TTL=$(cat accesstokenttl)", - "echo $WOPI_TTL", - "export WOPI_SRC=$(cat wopisrc)", - "echo $WOPI_SRC", - "cd /app", - "/app/Microsoft.Office.WopiValidator -t $WOPI_TOKEN -w $WOPI_SRC -l $WOPI_TTL --testgroup %s" % testgroup, - ], - }) - if wopiServerType == "builtin": - for builtinOnlyGroup in builtinOnlyTestGroups: - validatorTests.append({ - "name": "wopiValidatorTests-%s" % builtinOnlyGroup, - "image": OC_CI_WOPI_VALIDATOR, - "commands": [ - "export WOPI_TOKEN=$(cat accesstoken)", - "echo $WOPI_TOKEN", - "export WOPI_TTL=$(cat accesstokenttl)", - "echo $WOPI_TTL", - "export WOPI_SRC=$(cat wopisrc)", - "echo $WOPI_SRC", - "cd /app", - "/app/Microsoft.Office.WopiValidator -s -t $WOPI_TOKEN -w $WOPI_SRC -l $WOPI_TTL --testgroup %s" % builtinOnlyGroup, - ], - }) - - return { - "kind": "pipeline", - "type": "docker", - "name": "wopiValidatorTests-%s" % wopiServerType, - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + - restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + - fakeOffice() + - waitForServices("fake-office", ["fakeoffice:8080"]) + - ocisServer(storage, deploy_type = "wopi_validator", extra_server_environment = extra_server_environment) + - wopiServer + - waitForServices("wopi-fakeoffice", ["wopi-fakeoffice:9300"]) + - [ - { - "name": "prepare-test-file", - "image": OC_CI_ALPINE, - "environment": {}, - "commands": [ - "curl -v -X PUT '%s/remote.php/webdav/test.wopitest' -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin -D headers.txt" % OCIS_URL, - "cat headers.txt", - "export FILE_ID=$(cat headers.txt | sed -n -e 's/^.*Oc-Fileid: //p')", - "export URL=\"%s/app/open?app_name=FakeOffice&file_id=$FILE_ID\"" % OCIS_URL, - "export URL=$(echo $URL | tr -d '[:cntrl:]')", - "curl -v -X POST \"$URL\" -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin > open.json", - "cat open.json", - "cat open.json | jq .form_parameters.access_token | tr -d '\"' > accesstoken", - "cat open.json | jq .form_parameters.access_token_ttl | tr -d '\"' > accesstokenttl", - "echo -n 'http://wopi-fakeoffice:9300/wopi/files/' > wopisrc", - "cat open.json | jq .app_url | sed -n -e 's/^.*files%2F//p' | tr -d '\"' >> wopisrc", - ], - }, - ] + - validatorTests, - "depends_on": getPipelineNames(buildOcisBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - } - -def coreApiTestPipeline(ctx): - pipelines = [] - - with_remote_php = [False] - if ctx.build.event == "cron" or "full-ci" in ctx.build.title.lower(): - with_remote_php.append(True) - - defaults = { - "suites": {}, - "skip": False, - "extraEnvironment": {}, - "extraServerEnvironment": {}, - "storages": ["ocis"], - "emailNeeded": False, - "antivirusNeeded": False, - "tikaNeeded": False, - "withRemotePhp": with_remote_php, - "k8s": False, - } - test_dir = "%s/tests/acceptance" % dirs["base"] - - if "coreApiTests" not in config: - return pipelines - - for name, matrix in config["coreApiTests"].items(): - if "skip" not in matrix or not matrix["skip"]: - params = {} - for item in defaults: - params[item] = matrix[item] if item in matrix else defaults[item] - for storage in params["storages"]: - for run_with_remote_php in params["withRemotePhp"]: - filter_tags = "~@skipOnGraph&&~@skipOnOcis-%s-Storage" % ("OC" if storage == "owncloud" else "OCIS") - expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage.upper()) - run_on_k8s = params["k8s"] and ctx.build.event == "cron" - ocis_url = OCIS_URL - wrapper_url = "http://%s:5200" % OCIS_SERVER_NAME - - #################### - # SETUP STEPS # - #################### - setup_steps = skipIfUnchanged(ctx, "acceptance-tests") - if params["tikaNeeded"]: - setup_steps += waitForServices("tika", ["tika:9998"]) - if params["antivirusNeeded"]: - setup_steps += waitForServices("clamav", ["clamav:3310"]) - if params["emailNeeded"]: - setup_steps += waitForServices("email", ["%s:%s" % (EMAIL_SMTP_HOST, EMAIL_PORT)]) - - if run_on_k8s: - ocis_url = OCIS_URL_K8S - wrapper_url = "http://ociswrapper:5200" - - setup_steps += ocisServerK8s(antivirus = params["antivirusNeeded"], email = params["emailNeeded"]) - else: - setup_steps += restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") - setup_steps += ocisServer( - storage, - extra_server_environment = params["extraServerEnvironment"], - with_wrapper = True, - tika_enabled = params["tikaNeeded"], - volumes = ([stepVolumeOcisStorage]), - ) - - #################### - # SERVICES # - #################### - services = [] - if params["tikaNeeded"]: - services += tikaService() - if params["emailNeeded"]: - services += emailService() - if params["antivirusNeeded"]: - services += clamavService() - if run_on_k8s: - services += k3sCluster() - - #################### - # PIPELINE # - #################### - pipeline = { - "kind": "pipeline", - "type": "docker", - "name": "Core-API-%s%s%s" % (name, "-withRemotePhp" if run_with_remote_php else "", "-k8s" if run_on_k8s else ""), - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": setup_steps + - [ - { - "name": "run-api-tests", - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "environment": { - "TEST_SERVER_URL": ocis_url, - "OCIS_REVA_DATA_ROOT": "%s" % (dirs["ocisRevaDataRoot"] if storage == "owncloud" else ""), - "STORAGE_DRIVER": storage, - "BEHAT_FILTER_TAGS": filter_tags, - "BEHAT_SUITES": ",".join(params["suites"]), - "ACCEPTANCE_TEST_TYPE": "core-api", - "EXPECTED_FAILURES_FILE": expected_failures_file, - "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, - "OCIS_WRAPPER_URL": wrapper_url, - "WITH_REMOTE_PHP": run_with_remote_php, - "K8S": run_on_k8s, - }, - "commands": [ - # merge the expected failures - "" if run_with_remote_php else "cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file), - "make -C %s test-acceptance-api" % (dirs["base"]), - ], - }, - ] + - apiTestFailureLog() + - ([] if run_on_k8s else generateCoverageFromAPITest(ctx, name)), - "services": services, - "depends_on": [] if run_on_k8s else getPipelineNames(buildOcisBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - "volumes": [{ - "name": "storage", - "temp": {}, - }], - } - pipelines.append(pipeline) - return pipelines - -def e2eTestPipeline(ctx): - defaults = { - "skip": False, - "suites": [], - "xsuites": [], - "totalParts": 0, - "tikaNeeded": False, - "keycloakNeeded": False, - } - - extra_server_environment = { - "OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": "%s" % dirs["bannedPasswordList"], - "GRAPH_AVAILABLE_ROLES": "b1e2218d-eef8-4d4c-b82d-0f1a1b48f3b5,a8d5fe5e-96e3-418d-825b-534dbdf22b99,fb6c3e19-e378-47e5-b277-9732f9de6e21,58c63c02-1d89-4572-916a-870abc5a1b7d,2d00ce52-1fc2-4dbc-8b95-a73b73395f5a,1c996275-f1c9-4e71-abdf-a42f6495e960,312c0871-5ef7-4b3a-85b6-0e4074c64049,aa97fe03-7980-45ac-9e50-b325749fd7e6,63e64e19-8d43-42ec-a738-2b6af2610efa", - "FRONTEND_CONFIGURABLE_NOTIFICATIONS": "true", - } - - e2e_trigger = { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/**", - "refs/pull/**", - ], - } - - e2e_volumes = [{ - "name": "uploads", - "temp": {}, - }, { - "name": "configs", - "temp": {}, - }, { - "name": "gopath", - "temp": {}, - }] - - pipelines = [] - - if ("skip-e2e" in ctx.build.title.lower()): - return pipelines - - if (ctx.build.event == "tag"): - return pipelines - - for name, suite in config["e2eTests"].items(): - if "skip" in suite and suite["skip"]: - continue - - params = {} - for item in defaults: - params[item] = suite[item] if item in suite else defaults[item] - - e2e_args = "" - if params["totalParts"] > 0: - e2e_args = "--total-parts %d" % params["totalParts"] - elif params["suites"]: - e2e_args = "--suites %s" % ",".join(params["suites"]) - - # suites to skip - if params["xsuites"]: - e2e_args += " --xsuites %s" % ",".join(params["xsuites"]) - - test_environment = { - "BASE_URL_OCIS": OCIS_DOMAIN, - "HEADLESS": "true", - "RETRY": "1", - "REPORT_TRACING": "with-tracing" in ctx.build.title.lower(), - "SKIP_A11Y_TESTS": "true", - } - - # configs to setup ocis with keycloak - if params["keycloakNeeded"]: - extra_server_environment.update({ - "PROXY_AUTOPROVISION_ACCOUNTS": "true", - "PROXY_ROLE_ASSIGNMENT_DRIVER": "oidc", - "OCIS_OIDC_ISSUER": "https://keycloak:8443/realms/oCIS", - "PROXY_OIDC_REWRITE_WELLKNOWN": "true", - "WEB_OIDC_CLIENT_ID": "web", - "PROXY_USER_OIDC_CLAIM": "preferred_username", - "PROXY_USER_CS3_CLAIM": "username", - "OCIS_ADMIN_USER_ID": "", - "OCIS_EXCLUDE_RUN_SERVICES": "idp", - "GRAPH_ASSIGN_DEFAULT_USER_ROLE": "false", - "GRAPH_USERNAME_MATCH": "none", - "KEYCLOAK_DOMAIN": "keycloak:8443", - "PROXY_CSP_CONFIG_FILE_LOCATION": "%s/tests/config/drone/csp.yaml" % dirs["base"], - "IDM_CREATE_DEMO_USERS": False, - }) - test_environment.update({ - "KEYCLOAK": "true", - "KEYCLOAK_HOST": "keycloak:8443", - }) - - services = postgresService() if params["keycloakNeeded"] else [] - services += tikaService() if params["tikaNeeded"] else [] - - steps_before = \ - skipIfUnchanged(ctx, "e2e-tests") + \ - restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + \ - restoreWebCache() + \ - restoreWebPnpmCache() + \ - (waitForServices("tika", ["tika:9998"]) if params["tikaNeeded"] else []) + \ - (keycloakService() if params["keycloakNeeded"] else []) + \ - ocisServer(extra_server_environment = extra_server_environment, with_wrapper = not params["keycloakNeeded"], tika_enabled = params["tikaNeeded"], debug = False, external_idp = params["keycloakNeeded"]) - - step_e2e = { - "name": "e2e-tests", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "environment": test_environment, - "commands": [ - "cd %s/tests/e2e" % dirs["web"], - ], - } - - steps_after = uploadTracingResult(ctx) + \ - logTracingResults() - - if params["totalParts"]: - for index in range(params["totalParts"]): - run_part = index + 1 - run_e2e = {} - run_e2e.update(step_e2e) - run_e2e["commands"] = [ - "cd %s/tests/e2e" % dirs["web"], - "bash run-e2e.sh %s --run-part %d" % (e2e_args, run_part), - ] - pipelines.append({ - "kind": "pipeline", - "type": "docker", - "name": "e2e-tests-%s-%s" % (name, run_part), - "steps": steps_before + [run_e2e] + steps_after, - "depends_on": getPipelineNames(buildOcisBinaryForTesting(ctx) + buildWebCache(ctx)), - "trigger": e2e_trigger, - "volumes": e2e_volumes, - "services": services, - }) - else: - step_e2e["commands"].append("bash run-e2e.sh %s" % e2e_args) - pipelines.append({ - "kind": "pipeline", - "type": "docker", - "name": "e2e-tests-%s" % name, - "steps": steps_before + [step_e2e] + steps_after, - "depends_on": getPipelineNames(buildOcisBinaryForTesting(ctx) + buildWebCache(ctx)), - "trigger": e2e_trigger, - "volumes": e2e_volumes, - "services": services, - }) - - return pipelines - -def multiServiceE2ePipeline(ctx): - pipelines = [] - - defaults = { - "skip": False, - "suites": [], - "xsuites": [], - "tikaNeeded": False, - } - - e2e_trigger = { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - } - - if ("skip-e2e" in ctx.build.title.lower()): - return pipelines - - # run this pipeline only for cron jobs and full-ci PRs - if (not "full-ci" in ctx.build.title.lower() and ctx.build.event != "cron"): - return pipelines - - extra_server_environment = { - "OCIS_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": "%s" % dirs["bannedPasswordList"], - "OCIS_JWT_SECRET": "some-ocis-jwt-secret", - "OCIS_SERVICE_ACCOUNT_ID": "service-account-id", - "OCIS_SERVICE_ACCOUNT_SECRET": "service-account-secret", - "OCIS_EXCLUDE_RUN_SERVICES": "storage-users", - "OCIS_GATEWAY_GRPC_ADDR": "0.0.0.0:9142", - "SETTINGS_GRPC_ADDR": "0.0.0.0:9191", - "GATEWAY_STORAGE_USERS_MOUNT_ID": "storage-users-id", - "GRAPH_AVAILABLE_ROLES": "b1e2218d-eef8-4d4c-b82d-0f1a1b48f3b5,a8d5fe5e-96e3-418d-825b-534dbdf22b99,fb6c3e19-e378-47e5-b277-9732f9de6e21,58c63c02-1d89-4572-916a-870abc5a1b7d,2d00ce52-1fc2-4dbc-8b95-a73b73395f5a,1c996275-f1c9-4e71-abdf-a42f6495e960,312c0871-5ef7-4b3a-85b6-0e4074c64049,aa97fe03-7980-45ac-9e50-b325749fd7e6,63e64e19-8d43-42ec-a738-2b6af2610efa", - "FRONTEND_CONFIGURABLE_NOTIFICATIONS": "true", - } - - storage_users_environment = { - "OCIS_CORS_ALLOW_ORIGINS": "%s,https://%s:9201" % (OCIS_URL, OCIS_SERVER_NAME), - "STORAGE_USERS_JWT_SECRET": "some-ocis-jwt-secret", - "STORAGE_USERS_MOUNT_ID": "storage-users-id", - "STORAGE_USERS_SERVICE_ACCOUNT_ID": "service-account-id", - "STORAGE_USERS_SERVICE_ACCOUNT_SECRET": "service-account-secret", - "STORAGE_USERS_GATEWAY_GRPC_ADDR": "%s:9142" % OCIS_SERVER_NAME, - "STORAGE_USERS_EVENTS_ENDPOINT": "%s:9233" % OCIS_SERVER_NAME, - "STORAGE_USERS_DATA_GATEWAY_URL": "%s/data" % OCIS_URL, - "OCIS_CACHE_STORE": "nats-js-kv", - "OCIS_CACHE_STORE_NODES": "%s:9233" % OCIS_SERVER_NAME, - "MICRO_REGISTRY_ADDRESS": "%s:9233" % OCIS_SERVER_NAME, - } - storage_users1_environment = { - "STORAGE_USERS_GRPC_ADDR": "storageusers1:9157", - "STORAGE_USERS_HTTP_ADDR": "storageusers1:9158", - "STORAGE_USERS_DEBUG_ADDR": "storageusers1:9159", - "STORAGE_USERS_DATA_SERVER_URL": "http://storageusers1:9158/data", - } - for item in storage_users_environment: - storage_users1_environment[item] = storage_users_environment[item] - - storage_users2_environment = { - "STORAGE_USERS_GRPC_ADDR": "storageusers2:9157", - "STORAGE_USERS_HTTP_ADDR": "storageusers2:9158", - "STORAGE_USERS_DEBUG_ADDR": "storageusers2:9159", - "STORAGE_USERS_DATA_SERVER_URL": "http://storageusers2:9158/data", - } - for item in storage_users_environment: - storage_users2_environment[item] = storage_users_environment[item] - - storage_users_services = startOcisService("storage-users", "storageusers1", storage_users1_environment, [stepVolumeOcisStorage]) + \ - startOcisService("storage-users", "storageusers2", storage_users2_environment, [stepVolumeOcisStorage]) + \ - ocisHealthCheck("storage-users", ["storageusers1:9159", "storageusers2:9159"]) - - for _, suite in config["e2eMultiService"].items(): - if "skip" in suite and suite["skip"]: - continue - - params = {} - for item in defaults: - params[item] = suite[item] if item in suite else defaults[item] - - e2e_args = "" - if params["suites"]: - e2e_args = "--suites %s" % ",".join(params["suites"]) - - # suites to skip - if params["xsuites"]: - e2e_args += " --xsuites %s" % ",".join(params["xsuites"]) - - steps = \ - skipIfUnchanged(ctx, "e2e-tests") + \ - restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + \ - restoreWebCache() + \ - restoreWebPnpmCache() + \ - waitForServices("tika", ["tika:9998"]) + \ - ocisServer(extra_server_environment = extra_server_environment, tika_enabled = params["tikaNeeded"], debug = False) + \ - storage_users_services + \ - [{ - "name": "e2e-tests", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "environment": { - "BASE_URL_OCIS": OCIS_DOMAIN, - "HEADLESS": "true", - "RETRY": "1", - "SKIP_A11Y_TESTS": "true", - }, - "commands": [ - "cd %s/tests/e2e" % dirs["web"], - "bash run-e2e.sh %s" % e2e_args, - ], - }] + \ - uploadTracingResult(ctx) + \ - logTracingResults() - - pipelines.append({ - "kind": "pipeline", - "type": "docker", - "name": "e2e-tests-multi-service", - "steps": steps, - "depends_on": getPipelineNames(buildOcisBinaryForTesting(ctx) + buildWebCache(ctx)), - "trigger": e2e_trigger, - "volumes": [ - { - "name": "storage", - "temp": {}, - }, - pipelineVolumeGo, - ], - "services": tikaService(), - }) - return pipelines - -def uploadTracingResult(ctx): - return [{ - "name": "upload-tracing-result", - "image": PLUGINS_S3, - "pull": "if-not-exists", - "settings": { - "bucket": S3_PUBLIC_CACHE_BUCKET, - "endpoint": S3_CACHE_SERVER, - "path_style": True, - "source": "webTestRunner/reports/e2e/playwright/tracing/**/*", - "strip_prefix": "webTestRunner/reports/e2e/playwright/tracing", - "target": "/${DRONE_REPO}/${DRONE_BUILD_NUMBER}/tracing", - }, - "environment": { - "AWS_ACCESS_KEY_ID": { - "from_secret": "cache_public_s3_access_key", - }, - "AWS_SECRET_ACCESS_KEY": { - "from_secret": "cache_public_s3_secret_key", - }, - }, - "when": { - "status": [ - "failure", - ], - "event": [ - "pull_request", - "cron", - ], - }, - }] - -def logTracingResults(): - return [{ - "name": "log-tracing-result", - "image": OC_UBUNTU, - "commands": [ - "cd %s/reports/e2e/playwright/tracing/" % dirs["web"], - 'echo "To see the trace, please open the following link in the console"', - 'for f in *.zip; do echo "npx playwright show-trace https://cache.owncloud.com/public/${DRONE_REPO}/${DRONE_BUILD_NUMBER}/tracing/$f \n"; done', - ], - "when": { - "status": [ - "failure", - ], - "event": [ - "pull_request", - "cron", - ], - }, - }] - -def dockerReleases(ctx): - pipelines = [] - docker_repos = [] - build_type = "daily" - - # dockerhub repo - # - "owncloud/ocis-rolling" - repo = ctx.repo.slug + "-rolling" - docker_repos.append(repo) - - # production release repo - if ctx.build.event == "tag": - tag = ctx.build.ref.replace("refs/tags/v", "").lower() - for prod_tag in PRODUCTION_RELEASE_TAGS: - if tag.startswith(prod_tag): - docker_repos.append(ctx.repo.slug) - break - - for repo in docker_repos: - repo_pipelines = [] - if ctx.build.event == "tag": - build_type = "rolling" if "rolling" in repo else "production" - - for arch in config["dockerReleases"]["architectures"]: - repo_pipelines.append(dockerRelease(ctx, arch, repo, build_type)) - - manifest = releaseDockerManifest(ctx, repo, build_type) - manifest["depends_on"] = getPipelineNames(repo_pipelines) - repo_pipelines.append(manifest) - - readme = releaseDockerReadme(ctx, repo, build_type) - readme["depends_on"] = getPipelineNames(repo_pipelines) - repo_pipelines.append(readme) - - pipelines.extend(repo_pipelines) - - return pipelines - -def dockerRelease(ctx, arch, repo, build_type): - build_args = [ - "REVISION=%s" % (ctx.build.commit), - "VERSION=%s" % (ctx.build.ref.replace("refs/tags/", "") if ctx.build.event == "tag" else "master"), - ] - depends_on = getPipelineNames(testOcisAndUploadResults(ctx) + testPipelines(ctx)) - - if ctx.build.event == "tag": - depends_on = [] - - return { - "kind": "pipeline", - "type": "docker", - "name": "docker-%s-%s" % (arch, build_type), - "platform": { - "os": "linux", - "arch": arch, - }, - "steps": skipIfUnchanged(ctx, "build-docker") + - makeNodeGenerate("") + - makeGoGenerate("") + [ - { - "name": "build", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make -C ocis release-linux-docker-%s ENABLE_VIPS=true" % (arch), - ], - }, - { - "name": "dryrun", - "image": PLUGINS_DOCKER, - "settings": { - "dry_run": True, - "context": "ocis", - "tags": "linux-%s" % (arch), - "dockerfile": "ocis/docker/Dockerfile.linux.%s" % (arch), - "repo": repo, - "build_args": build_args, - }, - "when": { - "ref": { - "include": [ - "refs/pull/**", - ], - }, - }, - }, - { - "name": "docker", - "image": PLUGINS_DOCKER, - "settings": { - "username": { - "from_secret": "docker_username", - }, - "password": { - "from_secret": "docker_password", - }, - "auto_tag": True, - "context": "ocis", - "auto_tag_suffix": "linux-%s" % (arch), - "dockerfile": "ocis/docker/Dockerfile.linux.%s" % (arch), - "repo": repo, - "build_args": build_args, - }, - "when": { - "ref": { - "exclude": [ - "refs/pull/**", - ], - }, - }, - }, - ], - "depends_on": depends_on, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/v*", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - } - -def binaryReleases(ctx): - pipelines = [] - targets = [] - build_type = "daily" - - # uploads binary to https://download.owncloud.com/ocis/ocis/daily/ - target = "/ocis/%s/daily" % (ctx.repo.name.replace("ocis-", "")) - depends_on = getPipelineNames(testOcisAndUploadResults(ctx) + testPipelines(ctx)) - - if ctx.build.event == "tag": - depends_on = [] - - buildref = ctx.build.ref.replace("refs/tags/v", "").lower() - target_path = "/ocis/%s" % ctx.repo.name.replace("ocis-", "") - - if buildref.find("-") != -1: # "x.x.x-alpha", "x.x.x-beta", "x.x.x-rc" - folder = "testing" - target = "%s/%s/%s" % (target_path, folder, buildref) - targets.append(target) - build_type = "testing" - else: - # uploads binary to eg. https://download.owncloud.com/ocis/ocis/rolling/1.0.0/ - folder = "rolling" - target = "%s/%s/%s" % (target_path, folder, buildref) - targets.append(target) - - for prod_tag in PRODUCTION_RELEASE_TAGS: - if buildref.startswith(prod_tag): - # uploads binary to eg. https://download.owncloud.com/ocis/ocis/stable/2.0.0/ - folder = "stable" - target = "%s/%s/%s" % (target_path, folder, buildref) - targets.append(target) - break - - else: - targets.append(target) - - for target in targets: - if "rolling" in target: - build_type = "rolling" - elif "stable" in target: - build_type = "production" - elif "testing" in target: - build_type = "testing" - - for os in config["binaryReleases"]["os"]: - pipelines.append(binaryRelease(ctx, os, build_type, target, depends_on)) - - return pipelines - -def binaryRelease(ctx, arch, build_type, target, depends_on = []): - settings = { - "endpoint": { - "from_secret": "upload_s3_endpoint", - }, - "access_key": { - "from_secret": "upload_s3_access_key", - }, - "secret_key": { - "from_secret": "upload_s3_secret_key", - }, - "bucket": { - "from_secret": "upload_s3_bucket", - }, - "path_style": True, - "strip_prefix": "ocis/dist/release/", - "source": "ocis/dist/release/*", - "target": target, - } - - # [OCISDEV-266] Avoid cross-pipeline races on GitHub Release assets by: - # - Uploading only OS-scoped files from each pipeline - # - Copying and uploading the shared EULA only from the linux pipeline - artifacts = [] - if arch == "linux": - artifacts = [ - "ocis/dist/release/*-linux-*", - "ocis/dist/release/End-User-License-Agreement-for-ownCloud-Infinite-Scale.pdf", - ] - elif arch == "darwin": - artifacts = [ - "ocis/dist/release/*-darwin-*", - ] - - finish_commands = [ - "make -C ocis release-finish", - ] - if arch == "linux": - finish_commands.append("cp assets/End-User-License-Agreement-for-ownCloud-Infinite-Scale.pdf ocis/dist/release/") - - return { - "kind": "pipeline", - "type": "docker", - "name": "binaries-%s-%s" % (arch, build_type), - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "build-binary") + - makeNodeGenerate("") + - makeGoGenerate("") + [ - { - "name": "build", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make -C ocis release-%s" % (arch), - ], - }, - { - "name": "finish", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": finish_commands, - "when": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/v*", - ], - }, - }, - { - "name": "upload", - "image": PLUGINS_S3, - "settings": settings, - "when": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/v*", - ], - }, - }, - { - "name": "changelog", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make changelog CHANGELOG_VERSION=%s" % ctx.build.ref.replace("refs/tags/v", ""), - ], - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - { - "name": "release", - "image": PLUGINS_GITHUB_RELEASE, - "settings": { - "api_key": { - "from_secret": "github_token", - }, - "files": artifacts, - "title": ctx.build.ref.replace("refs/tags/v", ""), - "note": "ocis/dist/CHANGELOG.md", - "overwrite": True, - "prerelease": len(ctx.build.ref.split("-")) > 1, - }, - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - ], - "depends_on": depends_on, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/v*", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - } - -def licenseCheck(ctx): - # uploads third-party-licenses to https://download.owncloud.com/ocis/ocis/daily/ - target = "/ocis/%s/daily" % (ctx.repo.name.replace("ocis-", "")) - if ctx.build.event == "tag": - # uploads third-party-licenses to eg. https://download.owncloud.com/ocis/ocis/1.0.0-beta9/ - folder = "stable" - buildref = ctx.build.ref.replace("refs/tags/v", "") - buildref = buildref.lower() - if buildref.find("-") != -1: # "x.x.x-alpha", "x.x.x-beta", "x.x.x-rc" - folder = "testing" - target = "/ocis/%s/%s/%s" % (ctx.repo.name.replace("ocis-", ""), folder, buildref) - - settings = { - "endpoint": { - "from_secret": "upload_s3_endpoint", - }, - "access_key": { - "from_secret": "upload_s3_access_key", - }, - "secret_key": { - "from_secret": "upload_s3_secret_key", - }, - "bucket": { - "from_secret": "upload_s3_bucket", - }, - "path_style": True, - "source": "third-party-licenses.tar.gz", - "target": target, - } - - return [{ - "kind": "pipeline", - "type": "docker", - "name": "check-licenses", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "node-check-licenses", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "make ci-node-check-licenses", - ], - }, - { - "name": "node-save-licenses", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "make ci-node-save-licenses", - ], - }, - { - "name": "go-check-licenses", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make ci-go-check-licenses", - ], - "volumes": [stepVolumeGo], - }, - { - "name": "go-save-licenses", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make ci-go-save-licenses", - ], - "volumes": [stepVolumeGo], - }, - { - "name": "tarball", - "image": OC_CI_ALPINE, - "commands": [ - "cd third-party-licenses && tar -czf ../third-party-licenses.tar.gz *", - ], - }, - { - "name": "upload", - "image": PLUGINS_S3, - "settings": settings, - "when": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/v*", - ], - }, - }, - { - "name": "changelog", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make changelog CHANGELOG_VERSION=%s" % ctx.build.ref.replace("refs/tags/v", "").split("-")[0], - ], - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - { - "name": "release", - "image": PLUGINS_GITHUB_RELEASE, - "settings": { - "api_key": { - "from_secret": "github_token", - }, - "files": [ - "third-party-licenses.tar.gz", - ], - "title": ctx.build.ref.replace("refs/tags/v", ""), - "note": "ocis/dist/CHANGELOG.md", - "overwrite": True, - "prerelease": len(ctx.build.ref.split("-")) > 1, - }, - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/v*", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - }] - -def releaseDockerManifest(ctx, repo, build_type): - spec = "manifest.tmpl" - spec_latest = "manifest-latest.tmpl" - if "rolling" not in repo: - spec = "manifest.production.tmpl" - spec_latest = "manifest.production-latest.tmpl" - - steps = [ - { - "name": "execute", - "image": PLUGINS_MANIFEST, - "settings": { - "username": { - "from_secret": "docker_username", - }, - "password": { - "from_secret": "docker_password", - }, - "spec": "ocis/docker/%s" % spec, - "auto_tag": True if ctx.build.event == "tag" else False, - "ignore_missing": True, - }, - }, - ] - if len(ctx.build.ref.split("-")) == 1: - steps.append( - { - "name": "execute-latest", - "image": PLUGINS_MANIFEST, - "settings": { - "username": { - "from_secret": "docker_username", - }, - "password": { - "from_secret": "docker_password", - }, - "spec": "ocis/docker/%s" % spec_latest, - "auto_tag": True, - "ignore_missing": True, - }, - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - ) - - return { - "kind": "pipeline", - "type": "docker", - "name": "manifest-%s" % build_type, - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": steps, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/v*", - ], - }, - } - -def changelog(): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "changelog", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "generate", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make -C ocis changelog", - ], - }, - { - "name": "diff", - "image": OC_CI_ALPINE, - "commands": [ - "git diff", - ], - }, - { - "name": "output", - "image": OC_CI_ALPINE, - "commands": [ - "cat CHANGELOG.md", - ], - }, - { - "name": "publish", - "image": PLUGINS_GIT_ACTION, - "settings": { - "actions": [ - "commit", - "push", - ], - "message": "Automated changelog update [skip ci]", - "branch": "master", - "author_email": "devops@owncloud.com", - "author_name": "ownClouders", - "netrc_machine": "github.com", - "netrc_username": { - "from_secret": "github_username", - }, - "netrc_password": { - "from_secret": "github_token", - }, - }, - "when": { - "ref": { - "exclude": [ - "refs/pull/**", - ], - }, - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - "event": { - "exclude": [ - "cron", - ], - }, - }, - }] - -def releaseDockerReadme(ctx, repo, build_type): - return { - "kind": "pipeline", - "type": "docker", - "name": "readme-%s" % build_type, - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "execute", - "image": CHKO_DOCKER_PUSHRM, - "environment": { - "DOCKER_USER": { - "from_secret": "docker_username", - }, - "DOCKER_PASS": { - "from_secret": "docker_password", - }, - "PUSHRM_TARGET": repo, - "PUSHRM_SHORT": "Docker images for %s" % (ctx.repo.name), - "PUSHRM_FILE": "README.md", - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/v*", - ], - }, - } - -def documentation(): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "documentation", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "docs-generate", - "image": OC_CI_GOLANG, - # "pull": "always", - "environment": DRONE_HTTP_PROXY_ENV, - "commands": ["make docs-generate"], - }, - { - "name": "docs-copy", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": ["make docs-copy"], - }, - { - "name": "docs-hugo-drone-prep", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": ["make docs-hugo-drone-prep"], - }, - { - "name": "docs-build", - "image": OC_CI_HUGO_STATIC_IMAGE, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "cd hugo", - "hugo", - ], - }, - { - "name": "publish", - "image": PLUGINS_GH_PAGES, - "settings": { - "username": { - "from_secret": "github_username", - }, - "password": { - "from_secret": "github_token", - }, - "pages_directory": "docs/hugo/content/", - "copy_contents": "true", - "target_branch": "docs", - "delete": "true", - }, - "when": { - "ref": { - "exclude": [ - "refs/pull/**", - ], - }, - }, - }, - { - "name": "list and remove temporary files", - "image": OC_CI_ALPINE, - "commands": [ - "tree docs/hugo/public", - "rm -rf docs/hugo", - "rm -rf hugo", - ], - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - }] - -def makeNodeGenerate(module): - if module == "": - make = "make" - else: - make = "make -C %s" % (module) - return [ - { - "name": "generate nodejs", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "environment": { - "CHROMEDRIVER_SKIP_DOWNLOAD": "true", # install fails on arm and chromedriver is a test only dependency - }, - "commands": [ - "pnpm config set store-dir ./.pnpm-store", - "retry -t 3 '%s ci-node-generate'" % (make), - ], - "volumes": [stepVolumeGo], - }, - ] - -def makeGoGenerate(module): - if module == "": - make = "make" - else: - make = "make -C %s" % (module) - return [ - { - "name": "generate go", - "image": OC_CI_GOLANG, - "commands": [ - "retry -t 3 '%s ci-go-generate'" % (make), - ], - "environment": DRONE_HTTP_PROXY_ENV, - "volumes": [stepVolumeGo], - }, - ] - -def ocisServer(storage = "ocis", volumes = [], depends_on = [], deploy_type = "", extra_server_environment = {}, with_wrapper = False, tika_enabled = False, debug = True, external_idp = False): - user = "0:0" - container_name = OCIS_SERVER_NAME - environment = { - "OCIS_URL": OCIS_URL, - "OCIS_CONFIG_DIR": "/root/.ocis/config", # needed for checking config later - "STORAGE_USERS_DRIVER": "%s" % (storage), - "PROXY_ENABLE_BASIC_AUTH": True, - "WEB_UI_CONFIG_FILE": "%s/%s" % (dirs["base"], dirs["ocisConfig"]), - "OCIS_LOG_LEVEL": "error", - "IDM_CREATE_DEMO_USERS": True, # needed for litmus and cs3api-validator tests - "IDM_ADMIN_PASSWORD": "admin", # override the random admin password from `ocis init` - "FRONTEND_SEARCH_MIN_LENGTH": "2", - "OCIS_ASYNC_UPLOADS": True, - "OCIS_EVENTS_ENABLE_TLS": False, - "NATS_NATS_HOST": "0.0.0.0", - "NATS_NATS_PORT": 9233, - "OCIS_JWT_SECRET": "some-ocis-jwt-secret", - "EVENTHISTORY_STORE": "memory", - "OCIS_TRANSLATION_PATH": "%s/tests/config/translations" % dirs["base"], - # debug addresses required for running services health tests - "ACTIVITYLOG_DEBUG_ADDR": "0.0.0.0:9197", - "APP_PROVIDER_DEBUG_ADDR": "0.0.0.0:9165", - "APP_REGISTRY_DEBUG_ADDR": "0.0.0.0:9243", - "AUTH_BASIC_DEBUG_ADDR": "0.0.0.0:9147", - "AUTH_MACHINE_DEBUG_ADDR": "0.0.0.0:9167", - "AUTH_SERVICE_DEBUG_ADDR": "0.0.0.0:9198", - "CLIENTLOG_DEBUG_ADDR": "0.0.0.0:9260", - "EVENTHISTORY_DEBUG_ADDR": "0.0.0.0:9270", - "FRONTEND_DEBUG_ADDR": "0.0.0.0:9141", - "GATEWAY_DEBUG_ADDR": "0.0.0.0:9143", - "GRAPH_DEBUG_ADDR": "0.0.0.0:9124", - "GROUPS_DEBUG_ADDR": "0.0.0.0:9161", - "IDM_DEBUG_ADDR": "0.0.0.0:9239", - "IDP_DEBUG_ADDR": "0.0.0.0:9134", - "INVITATIONS_DEBUG_ADDR": "0.0.0.0:9269", - "NATS_DEBUG_ADDR": "0.0.0.0:9234", - "OCDAV_DEBUG_ADDR": "0.0.0.0:9163", - "OCM_DEBUG_ADDR": "0.0.0.0:9281", - "OCS_DEBUG_ADDR": "0.0.0.0:9114", - "POSTPROCESSING_DEBUG_ADDR": "0.0.0.0:9255", - "PROXY_DEBUG_ADDR": "0.0.0.0:9205", - "SEARCH_DEBUG_ADDR": "0.0.0.0:9224", - "SETTINGS_DEBUG_ADDR": "0.0.0.0:9194", - "SHARING_DEBUG_ADDR": "0.0.0.0:9151", - "SSE_DEBUG_ADDR": "0.0.0.0:9139", - "STORAGE_PUBLICLINK_DEBUG_ADDR": "0.0.0.0:9179", - "STORAGE_SHARES_DEBUG_ADDR": "0.0.0.0:9156", - "STORAGE_SYSTEM_DEBUG_ADDR": "0.0.0.0:9217", - "STORAGE_USERS_DEBUG_ADDR": "0.0.0.0:9159", - "THUMBNAILS_DEBUG_ADDR": "0.0.0.0:9189", - "USERLOG_DEBUG_ADDR": "0.0.0.0:9214", - "USERS_DEBUG_ADDR": "0.0.0.0:9145", - "WEB_DEBUG_ADDR": "0.0.0.0:9104", - "WEBDAV_DEBUG_ADDR": "0.0.0.0:9119", - "WEBFINGER_DEBUG_ADDR": "0.0.0.0:9279", - "GOCOVERDIR": "reports", - } - - if deploy_type == "": - # fonts map for txt thumbnails (including unicode support) - environment["THUMBNAILS_TXT_FONTMAP_FILE"] = "%s/tests/config/drone/fontsMap.json" % (dirs["base"]) - - if deploy_type == "cs3api_validator": - environment["GATEWAY_GRPC_ADDR"] = "0.0.0.0:9142" # make gateway available to cs3api-validator - environment["OCIS_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD"] = False - - if deploy_type == "wopi_validator": - environment["GATEWAY_GRPC_ADDR"] = "0.0.0.0:9142" # make gateway available to wopi server - environment["APP_PROVIDER_EXTERNAL_ADDR"] = "com.owncloud.api.app-provider" - environment["APP_PROVIDER_DRIVER"] = "wopi" - environment["APP_PROVIDER_WOPI_APP_NAME"] = "FakeOffice" - environment["APP_PROVIDER_WOPI_APP_URL"] = "http://fakeoffice:8080" - environment["APP_PROVIDER_WOPI_INSECURE"] = "true" - environment["APP_PROVIDER_WOPI_WOPI_SERVER_EXTERNAL_URL"] = "http://wopi-fakeoffice:9300" - environment["APP_PROVIDER_WOPI_FOLDER_URL_BASE_URL"] = OCIS_URL - - if deploy_type == "federation": - environment["OCIS_URL"] = OCIS_FED_URL - environment["PROXY_HTTP_ADDR"] = OCIS_FED_DOMAIN - container_name = FED_OCIS_SERVER_NAME - - if tika_enabled: - environment["FRONTEND_FULL_TEXT_SEARCH_ENABLED"] = True - environment["SEARCH_EXTRACTOR_TYPE"] = "tika" - environment["SEARCH_EXTRACTOR_TIKA_TIKA_URL"] = "http://tika:9998" - environment["SEARCH_EXTRACTOR_CS3SOURCE_INSECURE"] = True - - for item in extra_server_environment: - environment[item] = extra_server_environment[item] - - ocis_bin = "ocis/bin/ocis" - if debug: - ocis_bin = "ocis/bin/ocis-debug" - - build_and_run_commands = [ - "%s server" % ocis_bin, - ] - if with_wrapper: - build_and_run_commands = [ - "make -C %s build" % dirs["ocisWrapper"], - "%s/bin/ociswrapper serve --bin %s --url %s --admin-username admin --admin-password admin" % (dirs["ocisWrapper"], ocis_bin, environment["OCIS_URL"]), - ] - - wait_for_ocis = waitForOcis(container_name, environment["OCIS_URL"], depends_on) - if external_idp: - wait_for_ocis = waitForServices("ocis", [OCIS_DOMAIN]) - - commands = [ - "mkdir -p $GOCOVERDIR", - "%s init --insecure true" % ocis_bin, - "cat $OCIS_CONFIG_DIR/ocis.yaml", - "cp tests/config/drone/app-registry.yaml /root/.ocis/config/app-registry.yaml", - ] + (build_and_run_commands) - - return [{ - "name": container_name, - "image": OC_CI_GOLANG, - "detach": True, - "environment": environment, - "user": user, - "commands": commands, - "volumes": volumes, - "depends_on": depends_on, - }] + wait_for_ocis - -def startOcisService(service = None, name = None, environment = {}, volumes = []): - """ - Starts an OCIS service in a detached container. - - Args: - service (str): The name of the service to start. - name (str): The name of the container. - environment (dict): The environment variables to set in the container. - volumes (list): The volumes to mount in the container. - - Returns: - list: A list of pipeline steps to start the service. - """ - - if not service: - return [] - if not name: - name = service - - return [ - { - "name": name, - "image": OC_CI_GOLANG, - "detach": True, - "environment": environment, - "commands": [ - "ocis/bin/ocis-debug %s server" % service, - ], - "volumes": volumes, - }, - ] - -def ocisServerK8s(federation = False, antivirus = False, email = False, tika = False, wopi = False, auth_app = False): - name = OCIS_SERVER_NAME - url = OCIS_URL_K8S - fed_name = FED_OCIS_SERVER_NAME - fed_url = OCIS_FED_URL_K8S - - steps = [] - expose_ext_server_step = [] - external_servers = [] - - environment = { - "ENABLE_ANTIVIRUS": antivirus, - "ENABLE_EMAIL": email, - "ENABLE_TIKA": tika, - "ENABLE_WOPI": wopi, - "ENABLE_OCM": federation, - "ENABLE_AUTH_APP": auth_app, - } - - if antivirus: - external_servers.append(["clamav", 3310]) - if tika: - external_servers.append(["tika", 9998]) - if email: - external_servers.append([EMAIL_SMTP_HOST, EMAIL_SMTP_PORT]) - if wopi: - external_servers += [["collabora", 9980], ["fakeoffice", 8080], ["onlyoffice", 443]] - - if external_servers: - expose_ext_server_step = exposeExternalServersK8s(external_servers) - - steps += prepareChartsK8s(environment) - - steps += waitK3sCluster(name) + \ - (waitK3sCluster(fed_name) if federation else []) + \ - setupConfigMapsK8s(name) + \ - (setupConfigMapsK8s(fed_name) if federation else []) + \ - expose_ext_server_step + \ - deployOcisK8s(name) + \ - (deployOcisK8s(fed_name) if federation else []) + \ - streamOcisLogsK8s(name) + \ - (streamOcisLogsK8s(fed_name) if federation else []) + \ - waitForOcis(name, url) + \ - (waitForOcis(fed_name, fed_url) if federation else []) + \ - ociswrapperK8s(name) - return steps - -def redis(): - return [ - { - "name": "redis", - "image": REDIS, - }, - ] - -def redisForOCStorage(storage = "ocis"): - if storage == "owncloud": - return redis() - else: - return - -def build(): - return [ - { - "name": "build", - "image": OC_CI_GOLANG, - "commands": [ - "retry -t 3 'make -C ocis build ENABLE_VIPS=true'", - ], - "environment": DRONE_HTTP_PROXY_ENV, - "volumes": [stepVolumeGo], - }, - ] - -def buildDebug(): - return [ - { - "name": "build debug binary", - "image": OC_CI_GOLANG, - "commands": [ - "retry -t 3 'make -C ocis build-debug ENABLE_VIPS=true'", - ], - "environment": DRONE_HTTP_PROXY_ENV, - "volumes": [stepVolumeGo], - }, - ] - -def skipIfUnchanged(ctx, type): - if ("full-ci" in ctx.build.title.lower() or ctx.build.event == "tag" or ctx.build.event == "cron"): - return [] - - base = [ - "^.github/.*", - "^.vscode/.*", - "^changelog/.*", - "^docs/.*", - "^deployments/.*", - "CHANGELOG.md", - "CONTRIBUTING.md", - "LICENSE", - "README.md", - ] - unit = [ - ".*_test.go$", - ] - acceptance = [ - "^tests/acceptance/.*", - ] - - skip = [] - if type == "acceptance-tests" or type == "e2e-tests" or type == "lint": - skip = base + unit - elif type == "unit-tests": - skip = base + acceptance - elif type == "build-binary" or type == "build-docker" or type == "litmus": - skip = base + unit + acceptance - elif type == "cache": - skip = base - else: - return [] - - return [{ - "name": "skip-if-unchanged", - "image": OC_CI_DRONE_SKIP_PIPELINE, - "settings": { - "ALLOW_SKIP_CHANGED": skip, - }, - "when": { - "event": [ - "pull_request", - ], - }, - }] - -def example_deploys(ctx): - on_merge_deploy = [ - "ocis_full/master.yml", - "ocis_full/onlyoffice-master.yml", - ] - nightly_deploy = [ - "ocis_ldap/rolling.yml", - "ocis_keycloak/rolling.yml", - "ocis_full/production.yml", - "ocis_full/rolling.yml", - "ocis_full/onlyoffice-rolling.yml", - "ocis_full/s3-rolling.yml", - ] - - # if on master branch: - configs = on_merge_deploy - rebuild = "false" - - if ctx.build.event == "tag": - configs = nightly_deploy - rebuild = "false" - - if ctx.build.event == "cron": - configs = on_merge_deploy + nightly_deploy - rebuild = "true" - - deploys = [] - for config in configs: - deploys.append(deploy(ctx, config, rebuild)) - - return deploys - -def deploy(ctx, config, rebuild): - return { - "kind": "pipeline", - "type": "docker", - "name": "deploy_%s" % (config), - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "clone continuous deployment playbook", - "image": ALPINE_GIT, - "commands": [ - "cd deployments/continuous-deployment-config", - "git clone https://github.com/owncloud-devops/continuous-deployment.git", - ], - }, - { - "name": "deploy", - "image": OC_CI_DRONE_ANSIBLE, - "failure": "ignore", - "environment": { - "CONTINUOUS_DEPLOY_SERVERS_CONFIG": "../%s" % (config), - "REBUILD": "%s" % (rebuild), - "HCLOUD_API_TOKEN": { - "from_secret": "hcloud_api_token", - }, - "CLOUDFLARE_API_TOKEN": { - "from_secret": "cloudflare_api_token", - }, - }, - "settings": { - "playbook": "deployments/continuous-deployment-config/continuous-deployment/playbook-all.yml", - "galaxy": "deployments/continuous-deployment-config/continuous-deployment/requirements.yml", - "requirements": "deployments/continuous-deployment-config/continuous-deployment/py-requirements.txt", - "inventory": "localhost", - "private_key": { - "from_secret": "ssh_private_key", - }, - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/tags/v*", - ], - }, - } - -def checkStarlark(): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "check-starlark", - "steps": [ - { - "name": "format-check-starlark", - "image": OC_CI_BAZEL_BUILDIFIER, - "commands": [ - "buildifier --mode=check .drone.star", - ], - }, - { - "name": "show-diff", - "image": OC_CI_BAZEL_BUILDIFIER, - "commands": [ - "buildifier --mode=fix .drone.star", - "git diff", - ], - "when": { - "status": [ - "failure", - ], - }, - }, - ], - "depends_on": [], - "trigger": { - "ref": [ - "refs/pull/**", - ], - }, - }] - -def genericCache(name, action, mounts, cache_path): - rebuild = "false" - restore = "false" - if action == "rebuild": - rebuild = "true" - action = "rebuild" - else: - restore = "true" - action = "restore" - - step = { - "name": "%s_%s" % (action, name), - "image": PLUGINS_S3_CACHE, - "settings": { - "endpoint": MINIO_MC_ENV["MC_HOST"], - "rebuild": rebuild, - "restore": restore, - "mount": mounts, - "access_key": MINIO_MC_ENV["AWS_ACCESS_KEY_ID"], - "secret_key": MINIO_MC_ENV["AWS_SECRET_ACCESS_KEY"], - "filename": "%s.tar" % (name), - "path": cache_path, - "fallback_path": cache_path, - }, - } - return step - -def uploadAPITestCoverageReport(ctx): - sonar_env = { - "SONAR_TOKEN": { - "from_secret": "sonarcloud_acceptance_tests", - }, - } - - if ctx.build.event == "pull_request": - sonar_env.update({ - "SONAR_PULL_REQUEST_BASE": "%s" % (ctx.build.target), - "SONAR_PULL_REQUEST_BRANCH": "%s" % (ctx.build.source), - "SONAR_PULL_REQUEST_KEY": "%s" % (ctx.build.ref.replace("refs/pull/", "").split("/")[0]), - }) - - return { - "kind": "pipeline", - "type": "docker", - "name": "sonarcloud", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "sync-from-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "mkdir -p results", - "mc alias set cache $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc mirror cache/$CACHE_BUCKET/%s/%s/coverage results/" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - ], - }, - { - "name": "sonarcloud-properties", - "image": OC_UBUNTU, - "commands": [ - "mv sonar-project.properties sonar-project.properties.skip", - "mv tests/acceptance/sonar-project.properties sonar-project.properties", - ], - }, - { - "name": "sonarcloud", - "image": SONARSOURCE_SONAR_SCANNER_CLI, - "environment": sonar_env, - }, - { - "name": "purge-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "mc alias set cache $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc rm --recursive --force cache/$CACHE_BUCKET/%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - ], - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - ], - "status": [ - "success", - ], - }, - } - -def genericCachePurge(flush_path): - return { - "kind": "pipeline", - "type": "docker", - "name": "purge_build_artifact_cache", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "purge-cache", - "image": PLUGINS_S3_CACHE, - "settings": { - "access_key": MINIO_MC_ENV["AWS_ACCESS_KEY_ID"], - "secret_key": MINIO_MC_ENV["AWS_SECRET_ACCESS_KEY"], - "endpoint": MINIO_MC_ENV["MC_HOST"], - "flush": True, - "flush_age": 1, - "flush_path": flush_path, - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - "status": [ - "success", - "failure", - ], - }, - } - -def genericBuildArtifactCache(ctx, name, action, path): - if action == "rebuild" or action == "restore": - cache_path = "%s/%s/%s" % (S3_CACHE_BUCKET, ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}") - name = "%s_build_artifact_cache" % (name) - return genericCache(name, action, [path], cache_path) - - if action == "purge": - flush_path = "%s/%s" % (S3_CACHE_BUCKET, ctx.repo.slug) - return genericCachePurge(flush_path) - return [] - -def restoreBuildArtifactCache(ctx, name, path): - return [genericBuildArtifactCache(ctx, name, "restore", path)] - -def rebuildBuildArtifactCache(ctx, name, path): - return [genericBuildArtifactCache(ctx, name, "rebuild", path)] - -def purgeBuildArtifactCache(ctx): - return genericBuildArtifactCache(ctx, "", "purge", []) - -def pipelineSanityChecks(ctx, pipelines): - """pipelineSanityChecks helps the CI developers to find errors before running it - - These sanity checks are only executed on when converting starlark to yaml. - Error outputs are only visible when the conversion is done with the drone cli. - - Args: - ctx: drone passes a context with information which the pipeline can be adapted to - pipelines: pipelines to be checked, normally you should run this on the return value of main() - - Returns: - none - """ - - # check if name length of pipeline and steps are exceeded. - max_name_length = 50 - for pipeline in pipelines: - pipeline_name = pipeline["name"] - if len(pipeline_name) > max_name_length: - print("Error: pipeline name %s is longer than 50 characters" % (pipeline_name)) - - for step in pipeline["steps"]: - step_name = step["name"] - if len(step_name) > max_name_length: - print("Error: step name %s in pipeline %s is longer than 50 characters" % (step_name, pipeline_name)) - - # check for non existing depends_on - possible_depends = [] - for pipeline in pipelines: - possible_depends.append(pipeline["name"]) - - for pipeline in pipelines: - if "depends_on" in pipeline.keys(): - for depends in pipeline["depends_on"]: - if not depends in possible_depends: - print("Error: depends_on %s for pipeline %s is not defined" % (depends, pipeline["name"])) - - # check for non declared volumes - for pipeline in pipelines: - pipeline_volumes = [] - if "volumes" in pipeline.keys(): - for volume in pipeline["volumes"]: - pipeline_volumes.append(volume["name"]) - - for step in pipeline["steps"]: - if "volumes" in step.keys(): - for volume in step["volumes"]: - if not volume["name"] in pipeline_volumes: - print("Warning: volume %s for step %s is not defined in pipeline %s" % (volume["name"], step["name"], pipeline["name"])) - - # list used docker images - print("") - print("List of used docker images:") - - images = {} - - for pipeline in pipelines: - for step in pipeline["steps"]: - image = step["image"] - if image in images.keys(): - images[image] = images[image] + 1 - else: - images[image] = 1 - - for image in images.keys(): - print(" %sx\t%s" % (images[image], image)) - -def litmus(ctx, storage): - pipelines = [] - - if (config["litmus"] == False): - return pipelines - - environment = { - "LITMUS_PASSWORD": "admin", - "LITMUS_USERNAME": "admin", - "TESTS": "basic copymove props http", - } - - litmusCommand = "/usr/local/bin/litmus-wrapper" - - result = { - "kind": "pipeline", - "type": "docker", - "name": "litmus", - "workspace": { - "base": "/drone", - "path": "src", - }, - "steps": skipIfUnchanged(ctx, "litmus") + - restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + - ocisServer(storage) + - setupForLitmus() + - [ - { - "name": "old-endpoint", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - 'export LITMUS_URL="%s/remote.php/webdav"' % OCIS_URL, - litmusCommand, - ], - }, - { - "name": "new-endpoint", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - 'export LITMUS_URL="%s/remote.php/dav/files/admin"' % OCIS_URL, - litmusCommand, - ], - }, - { - "name": "new-shared", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - 'export LITMUS_URL="%s/remote.php/dav/files/admin/Shares/new_folder/"' % OCIS_URL, - litmusCommand, - ], - }, - { - "name": "old-shared", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - 'export LITMUS_URL="%s/remote.php/webdav/Shares/new_folder/"' % OCIS_URL, - litmusCommand, - ], - }, - # { - # "name": "public-share", - # "image": OC_LITMUS, - # "environment": { - # "LITMUS_PASSWORD": "admin", - # "LITMUS_USERNAME": "admin", - # "TESTS": "basic copymove http", - # }, - # "commands": [ - # "source .env", - # "export LITMUS_URL='%s/remote.php/dav/public-files/'$PUBLIC_TOKEN" % OCIS_URL, - # litmusCommand, - # ], - # }, - { - "name": "spaces-endpoint", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - "export LITMUS_URL='%s/remote.php/dav/spaces/'$SPACE_ID" % OCIS_URL, - litmusCommand, - ], - }, - ], - "services": redisForOCStorage(storage), - "depends_on": getPipelineNames(buildOcisBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - } - pipelines.append(result) - - return pipelines - -def setupForLitmus(): - return [{ - "name": "setup-for-litmus", - "image": OC_UBUNTU, - "environment": { - "TEST_SERVER_URL": OCIS_URL, - }, - "commands": [ - "bash ./tests/config/drone/setup-for-litmus.sh", - "cat .env", - ], - }] - -def getDroneEnvAndCheckScript(ctx): - ocis_git_base_url = "https://raw.githubusercontent.com/owncloud/ocis" - path_to_drone_env = "%s/%s/.drone.env" % (ocis_git_base_url, ctx.build.commit) - path_to_check_script = "%s/%s/tests/config/drone/check_web_cache.sh" % (ocis_git_base_url, ctx.build.commit) - return { - "name": "get-drone-env-and-check-script", - "image": OC_UBUNTU, - "commands": [ - "curl -s -o .drone.env %s" % path_to_drone_env, - "curl -s -o check_web_cache.sh %s" % path_to_check_script, - ], - } - -def checkForWebCache(name): - return { - "name": "check-for-%s-cache" % name, - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "bash -x check_web_cache.sh web.tar.gz", - ], - } - -def cloneWeb(): - return { - "name": "clone-web", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - ". ./.drone.env", - "rm -rf %s" % dirs["web"], - "git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/owncloud/web.git %s" % dirs["web"], - "cd %s && git checkout $WEB_COMMITID" % dirs["web"], - ], - } - -def generateWebPnpmCache(ctx): - return [ - getDroneEnvAndCheckScript(ctx), - checkForWebCache("web-pnpm"), - cloneWeb(), - { - "name": "install-pnpm", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "cd %s" % dirs["web"], - 'npm install --silent --global --force "$(jq -r ".packageManager" < package.json)"', - "pnpm config set store-dir ./.pnpm-store", - "retry -t 3 'pnpm install'", - ], - }, - { - "name": "zip-pnpm", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - # zip the pnpm deps before caching - "if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]), - "cd %s" % dirs["web"], - "tar -czvf %s .pnpm-store" % dirs["webPnpmZip"], - ], - }, - { - "name": "cache-pnpm", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "source ./.drone.env", - # cache using the minio/mc client to the public bucket (long term bucket) - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a %s s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID" % dirs["webPnpmZip"], - ], - }, - ] - -def generateWebCache(ctx): - return [ - getDroneEnvAndCheckScript(ctx), - checkForWebCache("web"), - cloneWeb(), - { - "name": "zip-web", - "image": OC_UBUNTU, - "commands": [ - "if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]), - "tar -czvf %s webTestRunner" % dirs["webZip"], - ], - }, - { - "name": "cache-web", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "source ./.drone.env", - # cache using the minio/mc client to the 'owncloud' bucket (long term bucket) - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a %s s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID" % dirs["webZip"], - ], - }, - ] - -def restoreWebCache(): - return [{ - "name": "restore-web-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "source ./.drone.env", - "rm -rf %s" % dirs["web"], - "mkdir -p %s" % dirs["web"], - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID/web.tar.gz %s" % dirs["zip"], - ], - }, { - "name": "unzip-web-cache", - "image": OC_UBUNTU, - "commands": [ - "tar -xvf %s -C ." % dirs["webZip"], - ], - }] - -def restoreWebPnpmCache(): - return [{ - "name": "restore-web-pnpm-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "source ./.drone.env", - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID/pnpm-store.tar.gz %s" % dirs["zip"], - ], - }, { - # we need to install again because the node_modules are not cached - "name": "unzip-and-install-pnpm", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "cd %s" % dirs["web"], - "rm -rf .pnpm-store", - "tar -xvf %s" % dirs["webPnpmZip"], - 'npm install --silent --global --force "$(jq -r ".packageManager" < package.json)"', - "pnpm config set store-dir ./.pnpm-store", - "retry -t 3 'pnpm install'", - ], - }] - -def emailService(): - return [{ - "name": EMAIL_SMTP_HOST, - "image": AXLLENT_MAILPIT, - }] - -def clamavService(): - return [{ - "name": "clamav", - "image": OC_CI_CLAMAVD, - }] - -def fakeOffice(): - return [ - { - "name": "fakeoffice", - "image": OC_CI_ALPINE, - "detach": True, - "environment": {}, - "commands": [ - "sh %s/tests/config/drone/serve-hosting-discovery.sh" % (dirs["base"]), - ], - }, - ] - -def wopiCollaborationService(name, ocis_url = OCIS_URL): - service_name = "wopi-%s" % name - - environment = { - "OCIS_URL": ocis_url, - "MICRO_REGISTRY": "nats-js-kv", - "MICRO_REGISTRY_ADDRESS": "%s:9233" % OCIS_SERVER_NAME, - "COLLABORATION_LOG_LEVEL": "debug", - "COLLABORATION_GRPC_ADDR": "0.0.0.0:9301", - "COLLABORATION_HTTP_ADDR": "0.0.0.0:9300", - "COLLABORATION_DEBUG_ADDR": "0.0.0.0:9304", - "COLLABORATION_APP_PROOF_DISABLE": "true", - "COLLABORATION_APP_INSECURE": "true", - "COLLABORATION_CS3API_DATAGATEWAY_INSECURE": "true", - "OCIS_JWT_SECRET": "some-ocis-jwt-secret", - "COLLABORATION_WOPI_SECRET": "some-wopi-secret", - } - - if name == "collabora": - environment["COLLABORATION_APP_NAME"] = "Collabora" - environment["COLLABORATION_APP_PRODUCT"] = "Collabora" - environment["COLLABORATION_APP_ADDR"] = "https://collabora:9980" - environment["COLLABORATION_APP_ICON"] = "https://collabora:9980/favicon.ico" - elif name == "onlyoffice": - environment["COLLABORATION_APP_NAME"] = "OnlyOffice" - environment["COLLABORATION_APP_PRODUCT"] = "OnlyOffice" - environment["COLLABORATION_APP_ADDR"] = "https://onlyoffice" - environment["COLLABORATION_APP_ICON"] = "https://onlyoffice/web-apps/apps/documenteditor/main/resources/img/favicon.ico" - elif name == "fakeoffice": - environment["COLLABORATION_APP_NAME"] = "FakeOffice" - environment["COLLABORATION_APP_PRODUCT"] = "Microsoft" - environment["COLLABORATION_APP_ADDR"] = "http://fakeoffice:8080" - - environment["COLLABORATION_WOPI_SRC"] = "http://%s:9300" % service_name - - return startOcisService("collaboration", service_name, environment) - -def tikaService(): - return [{ - "name": "tika", - "type": "docker", - "image": APACHE_TIKA, - }] - -def apiTestFailureLog(): - return [{ - "name": "api-test-failure-logs", - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "commands": [ - "cat %s/tests/acceptance/logs/failed.log" % dirs["base"], - ], - "when": { - "status": [ - "failure", - ], - }, - }] - -def k6LoadTests(ctx): - ocis_remote_environment = { - "SSH_OCIS_REMOTE": { - "from_secret": "k6_ssh_ocis_remote", - }, - "SSH_OCIS_USERNAME": { - "from_secret": "k6_ssh_ocis_user", - }, - "SSH_OCIS_PASSWORD": { - "from_secret": "k6_ssh_ocis_pass", - }, - "TEST_SERVER_URL": { - "from_secret": "k6_ssh_ocis_server_url", - }, - } - k6_remote_environment = { - "SSH_K6_REMOTE": { - "from_secret": "k6_ssh_k6_remote", - }, - "SSH_K6_USERNAME": { - "from_secret": "k6_ssh_k6_user", - }, - "SSH_K6_PASSWORD": { - "from_secret": "k6_ssh_k6_pass", - }, - } - environment = {} - environment.update(ocis_remote_environment) - environment.update(k6_remote_environment) - - if "skip" in config["k6LoadTests"] and config["k6LoadTests"]["skip"]: - return [] - - ocis_git_base_url = "https://raw.githubusercontent.com/owncloud/ocis" - script_link = "%s/%s/tests/config/drone/run_k6_tests.sh" % (ocis_git_base_url, ctx.build.commit) - - event_array = ["cron"] - trigger_ref = ["refs/heads/master"] - - if "k6-test" in ctx.build.title.lower(): - event_array.append("pull_request") - trigger_ref.append("refs/pull/**") - - return [{ - "kind": "pipeline", - "type": "docker", - "name": "k6-load-test", - "clone": { - "disable": True, - }, - "steps": [ - { - "name": "k6-load-test", - "image": OC_CI_ALPINE, - "environment": environment, - "commands": [ - "curl -s -o run_k6_tests.sh %s" % script_link, - "apk add --no-cache openssh-client sshpass", - "sh %s/run_k6_tests.sh" % (dirs["base"]), - ], - }, - { - "name": "ocis-log", - "image": OC_CI_ALPINE, - "environment": ocis_remote_environment, - "commands": [ - "curl -s -o run_k6_tests.sh %s" % script_link, - "apk add --no-cache openssh-client sshpass", - "sh %s/run_k6_tests.sh --ocis-log" % (dirs["base"]), - ], - "when": { - "status": [ - "success", - "failure", - ], - }, - }, - { - "name": "open-grafana-dashboard", - "image": OC_CI_ALPINE, - "commands": [ - "echo 'Grafana Dashboard: https://grafana.k6.infra.owncloud.works'", - ], - "when": { - "status": [ - "success", - "failure", - ], - }, - }, - ], - "depends_on": [], - "trigger": { - "event": event_array, - "ref": trigger_ref, - }, - }] - -def waitForOcis(name = "ocis", ocis_url = OCIS_URL, depends_on = []): - return [{ - "name": "wait-for-%s" % name, - "image": OC_CI_ALPINE, - "commands": [ - # wait for ocis-server to be ready (5 minutes) - "timeout 300 bash -c 'while [ $(curl -sk -uadmin:admin " + - "%s/graph/v1.0/users/admin " % ocis_url + - "-w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done'", - ], - "depends_on": depends_on, - }] - -def waitForServices(name, services = []): - services = ",".join(services) - return [{ - "name": "wait-for-%s" % name, - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it %s -t 300" % services, - ], - }] - -def ocisHealthCheck(name, services = []): - commands = [] - timeout = 300 - curl_command = ["timeout %s bash -c 'while [ $(curl -s %s/%s ", "-w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done'"] - for service in services: - commands.append(curl_command[0] % (timeout, service, "healthz") + curl_command[1]) - commands.append(curl_command[0] % (timeout, service, "readyz") + curl_command[1]) - - return [{ - "name": "health-check-%s" % name, - "image": OC_CI_ALPINE, - "commands": commands, - }] - -def collaboraService(): - return [ - { - "name": "collabora", - "type": "docker", - "image": COLLABORA_CODE, - "detach": True, - "environment": { - "DONT_GEN_SSL_CERT": "set", - "extra_params": "--o:ssl.enable=true --o:ssl.termination=true --o:welcome.enable=false --o:net.frame_ancestors=https://%s" % OCIS_SERVER_NAME, - }, - "commands": [ - "coolconfig generate-proof-key", - "bash /start-collabora-online.sh", - ], - }, - ] - -def onlyofficeService(): - return [ - { - "name": "onlyoffice", - "type": "docker", - "image": ONLYOFFICE_DOCUMENT_SERVER, - "detach": True, - "environment": { - "WOPI_ENABLED": "true", - "USE_UNAUTHORIZED_STORAGE": "true", # self signed certificates - }, - "commands": [ - "cp %s/tests/config/drone/only-office.json /etc/onlyoffice/documentserver/local.json" % dirs["base"], - "openssl req -x509 -newkey rsa:4096 -keyout onlyoffice.key -out onlyoffice.crt -sha256 -days 365 -batch -nodes", - "mkdir -p /var/www/onlyoffice/Data/certs", - "cp onlyoffice.key /var/www/onlyoffice/Data/certs/", - "cp onlyoffice.crt /var/www/onlyoffice/Data/certs/", - "chmod 400 /var/www/onlyoffice/Data/certs/onlyoffice.key", - "/app/ds/run-document-server.sh", - ], - }, - ] - -def keycloakService(): - return [{ - "name": "generate-keycloak-certs", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "cd %s" % dirs["base"], - "mkdir -p keycloak-certs", - "openssl req -x509 -newkey rsa:2048 -keyout keycloak-certs/keycloakkey.pem -out keycloak-certs/keycloakcrt.pem -nodes -days 365 -subj '/CN=keycloak'", - "chmod -R 777 keycloak-certs", - ], - }] + waitForServices("postgres", ["postgres:5432"]) + \ - [{ - "name": "keycloak", - "image": KEYCLOAK_IMAGE, - "detach": True, - "environment": { - "OCIS_DOMAIN": OCIS_URL, - "KC_HOSTNAME": "keycloak", - "KC_PORT": 8443, - "KC_DB": "postgres", - "KC_DB_URL": "jdbc:postgresql://postgres:5432/keycloak", - "KC_DB_USERNAME": "keycloak", - "KC_DB_PASSWORD": "keycloak", - "KC_FEATURES": "impersonation", - "KC_BOOTSTRAP_ADMIN_USERNAME": "admin", - "KC_BOOTSTRAP_ADMIN_PASSWORD": "admin", - "KC_HTTPS_CERTIFICATE_FILE": "%s/keycloak-certs/keycloakcrt.pem" % dirs["base"], - "KC_HTTPS_CERTIFICATE_KEY_FILE": "%s/keycloak-certs/keycloakkey.pem" % dirs["base"], - }, - "commands": [ - "mkdir -p /opt/keycloak/data/import", - "cp tests/config/drone/ocis-ci-realm.dist.json /opt/keycloak/data/import/oCIS-realm.json", - "/opt/keycloak/bin/kc.sh start-dev --proxy-headers xforwarded --spi-connections-http-client-default-disable-trust-manager=true --import-realm --health-enabled=true", - ], - }] + waitForServices("keycloak", ["keycloak:8443"]) - -def postgresService(): - return [ - { - "name": "postgres", - "image": POSTGRES_ALPINE_IMAGE, - "detach": True, - "environment": { - "POSTGRES_DB": "keycloak", - "POSTGRES_USER": "keycloak", - "POSTGRES_PASSWORD": "keycloak", - }, - }, - ] - -def trivyScan(ctx): - steps = [ - { - "name": "trivy-security-scan", - "image": TRIVY_IMAGE, - "commands": [ - # Trivy filesystem scan: recursively walks /drone/src workspace - # - Scans entire repository including compiled artifacts and dependencies - # - --exit-code 0: never fails the job regardless of findings - # - --no-progress: reduces log noise in CI - "trivy fs --exit-code 0 --severity HIGH,CRITICAL --no-progress /drone/src", - ], - "failure": "ignore", - }, - ] - - return { - "kind": "pipeline", - "type": "docker", - "name": "security-scan-trivy", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": steps, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/tags/**", - "refs/pull/**", - ], - }, - } - -def k3sCluster(name = OCIS_SERVER_NAME, ocm = False): - peer_name = OCIS_SERVER_NAME if name == FED_OCIS_SERVER_NAME else FED_OCIS_SERVER_NAME - commands = [] - log_dir = "%s/logs-%s" % (dirs["base"], name) - - commands.extend([ - "nohup dockerd-entrypoint.sh &", - "until docker ps 2>&1 > /dev/null; do sleep 1s; done", - # create ocis logs dir - "mkdir -p %s && chmod 777 %s" % (log_dir, log_dir), - # create cluster - "k3d cluster create %s --api-port %s:33199 " % (name, name) + - "-p '80:80@loadbalancer' -p '443:443@loadbalancer' -p '9100-9399:30100-30399@loadbalancer' " + - "--k3s-arg '--tls-san=k3d@server:*' --k3s-arg '--disable=metrics-server@server:*' " + - "-v %s:/logs" % log_dir, - # wait for services to be ready - "until kubectl get deployment coredns -n kube-system -o go-template='{{.status.availableReplicas}}' | grep -v -e ''; do sleep 1s; done", - "until kubectl get deployment traefik -n kube-system -o go-template='{{.status.availableReplicas}}' | grep -v -e ''; do sleep 1s; done", - "k3d kubeconfig get %s > kubeconfig-$${DRONE_BUILD_NUMBER}-%s.yaml" % (name, name), - "chmod 0600 kubeconfig-$${DRONE_BUILD_NUMBER}-%s.yaml" % name, - "printf '@@@@@@@@@@@@@@@@@@@@@@@\\n@@@@ k3d is ready @@@@\\n@@@@@@@@@@@@@@@@@@@@@@@\\n'", - # add dns rewrite rule - "kubectl create configmap coredns-custom -n kube-system " + - "--from-literal='rewritehost.override=rewrite name exact %s host.k3d.internal'" % name, - "kubectl -n kube-system rollout restart deployment coredns", - ]) - - if ocm: - commands.extend([ - # wait for peer - "until getent hosts %s >/dev/null 2>&1; do echo 'Waiting for %s...'; sleep 2; done" % (peer_name, peer_name), - # create namespace - "kubectl create namespace ocis || true", - "bash %s/tests/config/k8s/expose-external-svc.sh -n ocis %s:443" % (dirs["base"], peer_name), - ]) - - commands.extend([ - "printf '@@@@@@@@@@@@@@@@@@@@@@@\\n@@@@ k3d is ready @@@@\\n@@@@@@@@@@@@@@@@@@@@@@@\\n'", - # watch events - "kubectl get events -Aw", - ]) - - return [{ - "name": name, - "image": K3D_IMAGE, - "user": "root", - "privileged": True, - "commands": commands, - }] - -def waitK3sCluster(name = OCIS_SERVER_NAME): - step_name = "wait-cluster-" + name - return [{ - "name": step_name, - "image": K3D_IMAGE, - "user": "root", - "commands": [ - "export KUBECONFIG=kubeconfig-$${DRONE_BUILD_NUMBER}-%s.yaml" % name, - "timeout 300 bash -c 'until test -f $${KUBECONFIG}; do sleep 1; done'", - "kubectl config view", - "kubectl get pods -A", - ], - }] - -def prepareChartsK8s(environment = []): - commands = [ - # build ocis wrapper - "make -C %s build" % dirs["ocisWrapper"], - # get ocis charts - "[ ! -d %s/ocis-charts ] && " % dirs["base"] + - "git clone --single-branch -b main --depth 1 https://github.com/owncloud/ocis-charts.git", - # prepare charts for the tests - "bash %s/tests/config/k8s/setup.sh %s/ocis-charts" % (dirs["base"], dirs["base"]), - ] - - return [{ - "name": "prepare-ocis-deployment", - "image": OC_CI_GOLANG, - "environment": environment, - "commands": commands, - "volumes": [ - { - "name": "gopath", - "path": "/go", - }, - ], - }] - -def setupConfigMapsK8s(name = OCIS_SERVER_NAME): - step_name = "setup-configmaps-" + name - - commands = [ - "export KUBECONFIG=%s/kubeconfig-$${DRONE_BUILD_NUMBER}-%s.yaml" % (dirs["base"], name), - # Create namespace for oCIS deployment - "kubectl create namespace ocis || true", - # Setup Unicode font support for thumbnails - create ConfigMaps - "kubectl create configmap -n ocis ocis-fonts-ttf --from-file=%s/tests/config/drone/NotoSans.ttf" % dirs["base"], - ] - - return [{ - "name": step_name, - "image": K3D_IMAGE, - "user": "root", - "commands": commands, - }] - -def deployOcisK8s(name = OCIS_SERVER_NAME): - step_name = "deploy-" + name - - return [{ - "name": step_name, - "image": OC_CI_GOLANG, - "commands": [ - "export KUBECONFIG=%s/kubeconfig-$${DRONE_BUILD_NUMBER}-%s.yaml" % (dirs["base"], name), - "cd %s/ocis-charts" % dirs["base"], - # update external domain - "sed -i 's|externalDomain:.*|externalDomain: %s|' ./charts/ocis/ci/deployment-values.yaml" % name, - # deploy ocis - "make helm-install-atomic", - ], - }] - -def streamOcisLogsK8s(name = OCIS_SERVER_NAME): - return [{ - "name": "%s-logs" % name, - "image": OC_CI_ALPINE, - "detach": True, - "commands": [ - "until test -f logs-%s/ocis.log; do sleep 10s; done" % name, - "tail -f logs-%s/ocis.log" % name, - ], - }] - -def ociswrapperK8s(name = OCIS_SERVER_NAME, ocis_url = OCIS_URL_K8S): - return [{ - "name": "ociswrapper", - "image": K3D_IMAGE, - "detach": True, - "commands": [ - "export KUBECONFIG=kubeconfig-$${DRONE_BUILD_NUMBER}-%s.yaml" % name, - "until test -f $${KUBECONFIG}; do sleep 1s; done", - "%s/bin/ociswrapper serve --url %s --admin-username admin --admin-password admin --skip-ocis-run" % (dirs["ocisWrapper"], ocis_url), - ], - }, { - "name": "wait-for-wrapper", - "image": OC_CI_ALPINE, - "commands": [ - "timeout 60 bash -c 'while [ $(curl -sk " + - "http://ociswrapper:5200 " + - "-w %{http_code} -o /dev/null) != 404 ]; do sleep 1; done'", - ], - }] - -def exposeNodePortsK8s(services = [], name = OCIS_SERVER_NAME): - commands = [] - for idx, service in enumerate(services): - deploy_name = service[0] - service_port = str(service[1]) - node_port = service_port.replace("9", "30", 1) - - # there can be multiple collaboration services - if service_port == "9300" and "collaboration" in deploy_name: - node_port = "3030%d" % idx - service_name = "%s-%s-np" % (deploy_name, node_port) - commands.append("kubectl -n ocis expose deployment %s --type=NodePort --port=%s --name=%s" % (deploy_name, service_port, service_name)) - commands.append("kubectl -n ocis patch svc %s -p '{\"spec\":{\"ports\":[{\"port\":%s,\"nodePort\":%s}]}}'" % (service_name, service_port, node_port)) - - return [{ - "name": "expose-nodeports", - "image": K3D_IMAGE, - "commands": [ - "export KUBECONFIG=kubeconfig-$${DRONE_BUILD_NUMBER}-%s.yaml" % name, - "until test -f $${KUBECONFIG}; do sleep 1s; done", - ] + commands, - }] - -def exposeExternalServersK8s(servers = [], name = OCIS_SERVER_NAME): - servers_arg = [] - for server in servers: - servers_arg.append("%s:%s" % (server[0], str(server[1]))) - servers_arg = ",".join(servers_arg) - - return [{ - "name": "expose-external-servers", - "image": K3D_IMAGE, - "commands": [ - "export KUBECONFIG=kubeconfig-$${DRONE_BUILD_NUMBER}-%s.yaml" % name, - "until test -f $${KUBECONFIG}; do sleep 1s; done", - "bash %s/tests/config/k8s/expose-external-svc.sh -n ocis %s" % (dirs["base"], servers_arg), - ], - }] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 24953cd5d43..8e34fa4cac0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,3 @@ services/settings/ @lukashirt services/web/ @lukashirt -.drone.star @kobergj @phil-davis @individual-it assets/End-User-License-Agreement-for-ownCloud-Infinite-Scale.pdf @kobergj @hodyroff diff --git a/.github/release_template.md b/.github/release_template.md index 9d4adf7f4dc..211867c0489 100644 --- a/.github/release_template.md +++ b/.github/release_template.md @@ -8,7 +8,6 @@ * [ ] DEV/QA: Check new strings and align with clients * [ ] DEV/DOCS: Create list of pending docs tasks * [ ] DEV: Create branch `release-x.x.x-rc.x` -> CODEFREEZE - * [ ] DEV: specify if it is a production release in [.drone.star](https://github.com/owncloud/ocis/blob/b4cf38fa1ba180c58519026dfe762b7c45695466/.drone.star#L8) (if needed) * [ ] DEV: bump ocis version in necessary files * [ ] DEV: `changelog/CHANGELOG.tmpl` * [ ] DEV: `ocis-pkg/version/version.go` diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 1591250b122..167f57beb21 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -219,7 +219,7 @@ jobs: fail-fast: false matrix: suite: - - cliCommands,apiServiceAvailability # grouped like drone; needs ClamAV + email + - cliCommands,apiServiceAvailability # grouped: both need ClamAV + email services steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 @@ -375,7 +375,7 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.cache/ms-playwright - key: playwright-chromium-${{ hashFiles('.drone.env') }} + key: playwright-chromium-${{ hashFiles('ci.env') }} # --- Tika (search suite only) --- - name: Start Tika @@ -409,7 +409,8 @@ jobs: - name: Start Keycloak if: matrix.keycloak == true run: | - # Patch realm: replace Drone Docker hostname with localhost IP + # Patch realm: ocis-ci-realm.dist.json uses ocis-server:9200 as hostname. + # GitHub Actions runs on the host network so patch it to 127.0.0.1. sed 's|https://ocis-server:9200|https://127.0.0.1:9200|g' \ tests/config/drone/ocis-ci-realm.dist.json > /tmp/ocis-realm.json docker run -d --name keycloak --network host \ diff --git a/.gitignore b/.gitignore index 33e55761570..cacae4f6d18 100644 --- a/.gitignore +++ b/.gitignore @@ -45,8 +45,6 @@ vendor-php # QA activity reports tests/qa-activity-report/reports/ -# drone CI is in .drone.star, do not let someone accidentally commit a local .drone.yml -.drone.yml **/l10n/template.pot diff --git a/Makefile b/Makefile index 0da1c120f54..11e8752957a 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,6 @@ L10N_MODULES := \ services/userlog \ services/settings -# if you add a module here please also add it to the .drone.star file OCIS_MODULES = \ services/activitylog \ services/antivirus \ @@ -206,7 +205,10 @@ docs-serve: docs-clean: @$(MAKE) --no-print-directory -C docs docs-clean -# imitate a full drone run locally to build docs without pushing to the web. +# docs-local imitates the 3-step docs build pipeline (generate → copy → build) +# locally without pushing. docs-hugo-drone-prep creates the ../hugo symlink required by +# the Hugo Docker image (hugomods/hugo:base-0.129.0). +# imitate a full docs build run locally without pushing to the web. # this can help identify uncaught issues when running `make docs-serve` only. .PHONY: docs-local # run all steps as drone would do it (1, 2, 3) docs-local: @@ -365,9 +367,6 @@ l10n-write: $(MAKE) -C $$extension l10n-write || exit 1; \ done -.PHONY: ci-format -ci-format: $(BUILDIFIER) - $(BUILDIFIER) --mode=fix .drone.star .PHONY: test-php-style test-php-style: vendor-bin/owncloud-codestyle/vendor vendor-bin/php_codesniffer/vendor diff --git a/README.md b/README.md index 15ef8058263..0cbb5e56cbb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # ownCloud Infinite Scale [![Matrix](https://img.shields.io/matrix/ocis%3Amatrix.org?logo=matrix)](https://app.element.io/#/room/#ocis:matrix.org) -[![Build Status](https://drone.owncloud.com/api/badges/owncloud/ocis/status.svg)](https://drone.owncloud.com/owncloud/ocis) +[![Build Status](https://github.com/owncloud/ocis/actions/workflows/acceptance-tests.yml/badge.svg)](https://github.com/owncloud/ocis/actions/workflows/acceptance-tests.yml) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=owncloud_ocis&metric=security_rating)](https://sonarcloud.io/dashboard?id=owncloud_ocis) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=owncloud_ocis&metric=coverage)](https://sonarcloud.io/dashboard?id=owncloud_ocis) [![Acceptance Test Coverage](https://sonarcloud.io/api/project_badges/measure?project=owncloud-1_ocis_acceptance-tests&metric=coverage)](https://sonarcloud.io/summary/new_code?id=owncloud-1_ocis_acceptance-tests) diff --git a/ci.env b/ci.env new file mode 100644 index 00000000000..3c149ea0df4 --- /dev/null +++ b/ci.env @@ -0,0 +1,4 @@ +# Pins the ownCloud Web frontend version used in CI tests. +# Read by: tests/acceptance/run-e2e.py and .github/workflows/acceptance-tests.yml (Playwright cache key). +WEB_COMMITID=6165b4e47a47e3a216107e96f90825190f10632e +WEB_BRANCH=stable-12.3 diff --git a/docs/Makefile b/docs/Makefile index 271cbc8332b..fa4083db411 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -37,7 +37,8 @@ help: .PHONY:list list: help -# for drone only. drone uses an image for hugo that needs the hugo folder in the root +# Creates a symlink ../hugo → hugo/ required by the Hugo Docker image +# (hugomods/hugo:base-0.129.0), which expects the hugo folder at the repo root. # CURDIR will work for make only but not when running manually in the shell .PHONY: docs-hugo-drone-prep docs-hugo-drone-prep: diff --git a/docs/ocis/build-docs.md b/docs/ocis/build-docs.md index 25e4ae1523f..d703e48f7ed 100644 --- a/docs/ocis/build-docs.md +++ b/docs/ocis/build-docs.md @@ -15,7 +15,7 @@ The documentation: * Is based on the [HUGO](https://gohugo.io/) framework written in Go. The underlaying language is markdown plus the extensions provided by hugo. * The framwork including the theme used is maintained in the following [repo](https://github.com/owncloud/owncloud.github.io). -* The hugo version, currently at `0.129.0` must match the image version defined in the root makefile and the framework repo, defined in `.drone.yml`.\ +* The hugo version, currently at `0.129.0` must match the image version defined in the root makefile and the framework repo.\ {{< hint info >}} * Building the theme is part of the loaded framework while building the documentation is done using a separate container. @@ -24,7 +24,7 @@ The documentation: * The documentation sources are distributed over several repos such as `ocis`, `web` and others. When building, these sources act individually and only those sources are build and pushed to the web that come from the respective repo. The definition what is going where is defined in the [framework repo](https://github.com/owncloud/owncloud.github.io/blob/main/.batchfile). This means, each merge of a docs change in one of the sourcing repos only pushes that change to the site. There is no global build process and inter doc links must be treated therefore very carefully ! -* The documentation setup is, with only a few exceptions, identical for both the `ocis` and the `web` repo. This means, for example, running commands locally are the same, the documentation process section below applies for both; only drone uses some specifics. +* The documentation setup is, with only a few exceptions, identical for both the `ocis` and the `web` repo. This means, for example, running commands locally are the same, the documentation process section below applies for both. * {{< hint info >}} Any files or folders that need to be excluded from the build process must be defined in the referenced framework repository above, in the file `config.yaml`. diff --git a/docs/ocis/development/continuous-integration.md b/docs/ocis/development/continuous-integration.md index b5907d7cdc5..0034df1499a 100644 --- a/docs/ocis/development/continuous-integration.md +++ b/docs/ocis/development/continuous-integration.md @@ -9,48 +9,47 @@ geekdocFilePath: continuous-integration.md {{< toc >}} -oCIS uses [DRONE](https://www.drone.io/) as CI system. You can find the pipeline logs [here](https://drone.owncloud.com/owncloud/ocis) or in your PR. +## Overview + +oCIS uses [GitHub Actions](https://github.com/owncloud/ocis/actions) as its CI system. Pipeline logs are visible directly in pull requests. ## Concepts -The pipeline is defined in [Starlark](https://github.com/bazelbuild/starlark) and transformed to YAML upon pipeline run. This enables us to do a highly dynamic and non repeating pipeline configuration. We enforce Starlark format guidelines with Bazel Buildifier. You can format the .drone.star file by running `make ci-format`. +Pipelines are defined in `.github/workflows/`. The main acceptance test workflow is `.github/workflows/acceptance-tests.yml`. -Upon running the pipeline, your branch gets merged to the master branch. This ensures that we always test your changeset if as it was applied to the master of oCIS. Please note that this does not apply to the pipeline definition (`.drone.star`). +Upon running the pipeline, tests run against the branch as-is. Make sure your branch is rebased onto the latest master before running CI to avoid false failures. ## Things done in CI - static code analysis - linting - running UI tests -- running ownCloud 10 test suite against oCIS +- running API acceptance tests - build and release docker images - build and release binaries - build and release documentation ## Flags in commit message and PR title -You may add flags to your commit message or PR title in order to speed up pipeline runs and take load from the CI runners. +You may add flags to your commit message or PR title to control pipeline runs. - `[CI SKIP]`: no CI is run on the commit or PR - -- `[full-ci]`: deactivates the fail early mechanism and runs all available test (as default only smoke tests are run) +- `[full-ci]`: deactivates the fail-early mechanism and runs all available tests (by default only smoke tests run) ### Knowledge base - My pipeline fails because some CI related files or commands are missing. - Please make sure to rebase your branch onto the latest master of oCIS. It could be that the pipeline definition (`.drone.star`) was changed on the master branch. This is the only file, that will not be auto merged to master upon pipeline run. So things could be out of sync. + Make sure your branch is rebased onto the latest master. Workflow files in `.github/workflows/` change over time and a stale branch may reference steps or configs that no longer exist. -- How can I see the YAML drone pipeline definition? +- How can I see what a workflow does? - In order to see the Yaml pipeline definition you can use the drone-cli to convert the Starlark file. + Workflow definitions are plain YAML in `.github/workflows/`. Open the relevant file directly in the repo — no CLI tool needed. The main acceptance test entry point is `.github/workflows/acceptance-tests.yml`. - ``` - drone starlark - ``` +- How can I re-run a failed job? - {{< hint info >}} - If you experience a `"build" struct has no .title attribute` you need a newer version of drone-cli. + In the GitHub Actions UI, click **Re-run jobs** → **Re-run failed jobs** on the failed workflow run. You can also trigger a full re-run from `gh`: - You currently need to build it yourself from this [source code](https://github.com/drone/drone-cli). If you are not using master as source, please ensure that this [PR](https://github.com/drone/drone-cli/pull/175) is included. - {{< /hint >}} + ```bash + gh run rerun --failed --repo owncloud/ocis + ``` diff --git a/docs/ocis/releasing_guide/releasing_guide.md b/docs/ocis/releasing_guide/releasing_guide.md index 842df817cef..16714d6392f 100644 --- a/docs/ocis/releasing_guide/releasing_guide.md +++ b/docs/ocis/releasing_guide/releasing_guide.md @@ -108,10 +108,7 @@ https://owncloud.dev/ocis/releasing_guide/ * [ ] Create orphan branch: `git checkout --orphan docs-stable-7.2` * [ ] Initial commit: `git commit --allow-empty -m "initial commit"` * [ ] Push it: `git push` - * [ ] Adjust the `.drone.star` file to write to `docs-stable-7.2` - * [ ] Find target_branch value in the docs section and change it to `docs-stable-7.2` - * [ ] Example: https://github.com/owncloud/ocis/blame/56f7645f0b11c9112e15ce46f6effd2fea01d6be/.drone.star#L2249 - * [ ] Add `stable-7.2` to the nightly cron jobs in drone (`Settings` -> `Cron Jobs`) + * [ ] Add `stable-7.2` to the nightly cron jobs in GitHub Actions (`Settings` -> `Schedules`) ``` ### Steps for Patch Releases Only diff --git a/docs/services/general-info/add-translations.md b/docs/services/general-info/add-translations.md index 49c06809b20..802c182079f 100644 --- a/docs/services/general-info/add-translations.md +++ b/docs/services/general-info/add-translations.md @@ -16,12 +16,7 @@ Services can have texts that need to be translated. These translations will be s The process for _synchronisation_ with Transifex is already setup and nothing needs to be done here. For any translation, it is necessary to set it up in the respective service and tell to sync it. **IMPORTANT**\ -Translations are automatically synced on a daily basis in the night. To do so, there is an own repo that covers the process for ALL translations from all configured repos: [translation-sync](https://github.com/owncloud/translation-sync). If there is a manual "emergency" sync necessary, you only need to trigger [drone](https://drone.owncloud.com/owncloud/translation-sync) via cli - -```bash -drone cron exec owncloud/translation-sync nightly -``` -Note that you need to be logged on in drone to execute the command. +Translations are automatically synced on a daily basis in the night. To do so, there is an own repo that covers the process for ALL translations from all configured repos: [translation-sync](https://github.com/owncloud/translation-sync). If there is a manual "emergency" sync necessary, trigger the workflow manually via the [GitHub Actions UI](https://github.com/owncloud/translation-sync/actions) using `workflow_dispatch`. ## Implementing ocis Translations diff --git a/docs/services/general-info/new-service-checklist.md b/docs/services/general-info/new-service-checklist.md index eb72f9ad473..b2657c42a75 100644 --- a/docs/services/general-info/new-service-checklist.md +++ b/docs/services/general-info/new-service-checklist.md @@ -36,7 +36,7 @@ Use this checklist with copy/paste in your PR - right from the beginning. It ren - Add the service config to `ocis-pkg/config/defaultconfig.go` - [ ] If the service is using service accounts, add it to `ocis/pkg/init/init.go` - [ ] Check that the service properly responds to `ocis health` and has `/healthz` and `/readyz` endpoints -- [ ] Add the service to `.drone.star` to enable CI. +- [ ] Add the service suite to the GitHub Actions matrix in `.github/workflows/acceptance-tests.yml` to enable CI. - [ ] Inform doc team in an *early stage* to review the readme AND the environment variables created. - The description must reflect the behaviour AND usually has a positive code quality impact. - [ ] Create proper description strings for envvars - see other services for examples, especially when it comes to multiple values. This must include: diff --git a/docs/services/web/releasing.md b/docs/services/web/releasing.md index ac9be9f12f1..b7e09262aea 100644 --- a/docs/services/web/releasing.md +++ b/docs/services/web/releasing.md @@ -32,6 +32,6 @@ and take note of its release tag name. 2. Change into web package folder via `cd web` 3. Inside `web/`, update the `Makefile` so that the WEB_ASSETS_VERSION variable references the currently released version of https://github.com/owncloud/web 4. Move to the changelog (`cd ../changelog/`) and add a changelog file to the `unreleased/` folder (You can copy an old web release changelog item as a template) -5. Move to the repo root (`cd ..`)and update the WEB_COMMITID in the `/.drone.env` file to the commit id from the released version (unless the existing commit id is already newer) +5. Move to the repo root (`cd ..`) and update the `WEB_COMMITID` in `/ci.env` to the commit id from the released version (unless the existing commit id is already newer) 6. **Optional:** Test the changes locally by running `cd ocis && go run cmd/ocis/main.go server`, visiting [https://localhost:9200](https://localhost:9200) and confirming everything renders correctly 7. Commit your changes, push them and [create a PR](https://github.com/owncloud/ocis/pulls) diff --git a/go.mod b/go.mod index 107398b86e8..669816983e9 100644 --- a/go.mod +++ b/go.mod @@ -96,7 +96,7 @@ require ( go.opentelemetry.io/otel/trace v1.43.0 golang.org/x/crypto v0.50.0 golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f - golang.org/x/image v0.38.0 + golang.org/x/image v0.39.0 golang.org/x/net v0.53.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.20.0 diff --git a/go.sum b/go.sum index 391efea787c..b7c43bda1ae 100644 --- a/go.sum +++ b/go.sum @@ -1039,8 +1039,8 @@ golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aI golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= -golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= -golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= +golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww= +golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/tests/acceptance/docker/src/acceptance.yml b/tests/acceptance/docker/src/acceptance.yml index 732864fe9d9..43d85163d18 100644 --- a/tests/acceptance/docker/src/acceptance.yml +++ b/tests/acceptance/docker/src/acceptance.yml @@ -17,7 +17,7 @@ services: EMAIL_HOST: email EMAIL_PORT: 8025 env_file: - - ../../../../.drone.env + - ../../../../ci.env volumes: - ./run-tests.sh:/test/run-tests.sh - ../../../../:/drone/src diff --git a/tests/acceptance/docker/src/run-tests.sh b/tests/acceptance/docker/src/run-tests.sh index 5465c9bc81f..d2304c3b0ec 100644 --- a/tests/acceptance/docker/src/run-tests.sh +++ b/tests/acceptance/docker/src/run-tests.sh @@ -1,4 +1,7 @@ #!/bin/bash +# DRONE: Hardcodes /drone/src as the repo root (Drone workspace mount point). +# Lines 3-4 (commented out) and line 15 reference this path. +# When migrating: replace /drone/src with the appropriate env var or argument. #mkdir -p /drone/src/vendor-bin/behat #cp /tmp/vendor-bin/behat/composer.json /drone/src/vendor-bin/behat/composer.json diff --git a/tests/acceptance/run-cs3api.py b/tests/acceptance/run-cs3api.py index 9da46cb0888..a09f0ebbcca 100644 --- a/tests/acceptance/run-cs3api.py +++ b/tests/acceptance/run-cs3api.py @@ -2,7 +2,6 @@ """ Run CS3 API validator tests locally and in GitHub Actions CI. -Config sourced from .drone.star cs3ApiTests() — single source of truth. Usage: python3 tests/acceptance/run-cs3api.py """ @@ -15,7 +14,7 @@ from pathlib import Path # --------------------------------------------------------------------------- -# Constants (mirroring .drone.star) +# Constants # --------------------------------------------------------------------------- # HTTPS — matching drone: ocis init generates a self-signed cert; proxy uses TLS by default. diff --git a/tests/acceptance/run-e2e.py b/tests/acceptance/run-e2e.py index c8d6995fe72..95553bfd052 100755 --- a/tests/acceptance/run-e2e.py +++ b/tests/acceptance/run-e2e.py @@ -73,15 +73,15 @@ def main() -> int: # clone + install web only if not already provided (e.g. via artifact) if not web_dir.exists(): - drone_env = {} - drone_env_file = repo_root / ".drone.env" - if drone_env_file.exists(): - for line in drone_env_file.read_text().splitlines(): + ci_env = {} + ci_env_file = repo_root / "ci.env" + if ci_env_file.exists(): + for line in ci_env_file.read_text().splitlines(): if "=" in line and not line.startswith("#"): k, v = line.split("=", 1) - drone_env[k.strip()] = v.strip() - web_branch = drone_env.get("WEB_BRANCH", "master") - web_commitid = drone_env.get("WEB_COMMITID", "") + ci_env[k.strip()] = v.strip() + web_branch = ci_env.get("WEB_BRANCH", "master") + web_commitid = ci_env.get("WEB_COMMITID", "") run(["git", "clone", "-b", web_branch, "--single-branch", "--no-tags", WEB_REPO, str(web_dir)]) if web_commitid: diff --git a/tests/acceptance/run-github.py b/tests/acceptance/run-github.py index d2010ca07db..98cfcb03531 100755 --- a/tests/acceptance/run-github.py +++ b/tests/acceptance/run-github.py @@ -2,7 +2,6 @@ """ Run ocis acceptance tests locally and in GitHub Actions CI. -Config sourced from .drone.star localApiTests — single source of truth. Usage: BEHAT_SUITES=apiGraph python3 tests/acceptance/run-github.py """ @@ -17,7 +16,7 @@ from pathlib import Path # --------------------------------------------------------------------------- -# Config sourced from .drone.star + # NOTE: EMAIL_SMTP_HOST is "email" (container name) in drone, "localhost" here # --------------------------------------------------------------------------- diff --git a/tests/acceptance/run-litmus.py b/tests/acceptance/run-litmus.py index 631f2e7f94f..7835ad52403 100644 --- a/tests/acceptance/run-litmus.py +++ b/tests/acceptance/run-litmus.py @@ -2,7 +2,6 @@ """ Run litmus WebDAV compliance tests locally and in GitHub Actions CI. -Config sourced from .drone.star litmus() / setupForLitmus() — single source of truth. Usage: python3 tests/acceptance/run-litmus.py """ @@ -17,7 +16,7 @@ from pathlib import Path # --------------------------------------------------------------------------- -# Constants (mirroring .drone.star) +# Constants # --------------------------------------------------------------------------- # HTTPS — matching drone: ocis init generates a self-signed cert; proxy uses TLS by default. diff --git a/tests/acceptance/run-wopi.py b/tests/acceptance/run-wopi.py index 93607032bb2..8bb34ccf0b8 100644 --- a/tests/acceptance/run-wopi.py +++ b/tests/acceptance/run-wopi.py @@ -2,7 +2,6 @@ """ Run WOPI validator tests locally and in GitHub Actions CI. -Config sourced from .drone.star wopiValidatorTests() — single source of truth. Usage: python3 tests/acceptance/run-wopi.py --type builtin python3 tests/acceptance/run-wopi.py --type cs3 """ @@ -21,7 +20,7 @@ from pathlib import Path # --------------------------------------------------------------------------- -# Constants (mirroring .drone.star) +# Constants # --------------------------------------------------------------------------- # HTTPS — matching drone; host-side curl calls use -k. diff --git a/tests/acceptance/run.sh b/tests/acceptance/run.sh index 29de2cc6f1f..ead4c9096ba 100755 --- a/tests/acceptance/run.sh +++ b/tests/acceptance/run.sh @@ -677,6 +677,7 @@ then ${RED_COLOR}; printf "%s\n" "${UNEXPECTED_BEHAT_EXIT_STATUSES[@]}" fi +# DRONE: Dead code — workarounds for Drone log-flushing issues. Kept as archaeology. # # sync the file-system so all output will be flushed to storage. # # In drone we sometimes see that the last lines of output are missing from the # # drone log. diff --git a/tests/config/drone/check_go_bin_cache.sh b/tests/config/drone/check_go_bin_cache.sh index dac1e142984..5a529e20973 100644 --- a/tests/config/drone/check_go_bin_cache.sh +++ b/tests/config/drone/check_go_bin_cache.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +# DRONE: Checks S3 MinIO cache for Go binary artifacts (.bingo tools). +# Defaults root path to /drone/src (Drone workspace mount point). +# Exit code 78 is the Drone "skip pipeline without failure" signal. +# Used by .drone.star get-go-bin-cache pipeline step. # # $1 - root path where .bingo resides diff --git a/tests/config/drone/check_web_cache.sh b/tests/config/drone/check_web_cache.sh index 752dac0470b..977d26806fb 100644 --- a/tests/config/drone/check_web_cache.sh +++ b/tests/config/drone/check_web_cache.sh @@ -1,5 +1,7 @@ #!/bin/bash -source .drone.env +# Checks S3 MinIO cache for web test runner artifacts (acceptance/e2e/web). +# Sources ci.env to get WEB_COMMITID used as the cache key. +source ci.env # if no $1 is supplied end the script # Can be web, acceptance or e2e diff --git a/tests/config/drone/serve-hosting-discovery.sh b/tests/config/drone/serve-hosting-discovery.sh index b5975786659..03f80944455 100644 --- a/tests/config/drone/serve-hosting-discovery.sh +++ b/tests/config/drone/serve-hosting-discovery.sh @@ -1,4 +1,8 @@ #!/bin/sh +# DRONE: Serves a fake WOPI hosting discovery XML response on port 8080. +# Hardcodes /drone/src as the repo root (Drone workspace mount point). +# Used by .drone.star wopiValidatorTests pipeline step for FakeOffice WOPI testing. +# When migrating: replace /drone/src with the appropriate path or an env var. while true; do echo -e "HTTP/1.1 200 OK\n\n$(cat /drone/src/tests/config/drone/hosting-discovery.xml)" | nc -l -k -p 8080 diff --git a/tests/config/k8s/Makefile b/tests/config/k8s/Makefile index ba63bc7d4cc..e581fdad8e0 100644 --- a/tests/config/k8s/Makefile +++ b/tests/config/k8s/Makefile @@ -25,7 +25,7 @@ prepare-charts: @test -d "$(CHART_REPO)" || git clone --single-branch -b main --depth 1 https://github.com/owncloud/ocis-charts.git $(CHART_REPO) bash setup.sh $(CHART_REPO) kubectl create namespace $(DOMAIN) || true - kubectl create configmap -n $(DOMAIN) ocis-fonts-ttf --from-file=../drone/NotoSans.ttf || true + kubectl create configmap -n $(DOMAIN) ocis-fonts-ttf --from-file=../drone/NotoSans.ttf || true # DRONE: NotoSans.ttf lives in tests/config/drone/ — move if that directory is removed .PHONY: deploy-ocis deploy-ocis: diff --git a/vendor/github.com/RoaringBitmap/roaring/v2/.drone.yml b/vendor/github.com/RoaringBitmap/roaring/v2/.drone.yml new file mode 100644 index 00000000000..7936bfe8dfd --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/v2/.drone.yml @@ -0,0 +1,19 @@ +kind: pipeline +name: default + +workspace: + base: /go + path: src/github.com/RoaringBitmap/roaring + +steps: +- name: test + image: golang + commands: + - go get -t + - go test + - go build -tags appengine + - go test -tags appengine + - GOARCH=386 go build + - GOARCH=386 go test + - GOARCH=arm go build + - GOARCH=arm64 go build diff --git a/vendor/golang.org/x/image/font/sfnt/sfnt.go b/vendor/golang.org/x/image/font/sfnt/sfnt.go index 8ed19e21a9f..d1ef8a6a084 100644 --- a/vendor/golang.org/x/image/font/sfnt/sfnt.go +++ b/vendor/golang.org/x/image/font/sfnt/sfnt.go @@ -214,8 +214,9 @@ func u32(b []byte) uint32 { // copying from the source to a caller-supplied buffer, and instead provide // direct access to the underlying []byte data. type source struct { - b []byte - r io.ReaderAt + b []byte + r io.ReaderAt + minSize int // r is known to contain at least minSize bytes // TODO: add a caching layer, if we're using the io.ReaderAt? Note that // this might make a source no longer safe to use concurrently. @@ -255,6 +256,17 @@ func (s *source) view(buf []byte, offset, length int) ([]byte, error) { return s.b[offset : offset+length], nil } + if end := offset + length; end > s.minSize && length > 1<<20 { + // We're reading more than 1MiB, and we don't know whether + // the file contains this data. Check that the data exists + // before we try to allocate. + var oneByte [1]byte + if n, err := s.r.ReadAt(oneByte[:], int64(end)-1); err != nil || n != 1 { + return nil, errInvalidBounds + } + s.minSize = end + } + // Read from the io.ReaderAt. if length <= cap(buf) { buf = buf[:length] diff --git a/vendor/golang.org/x/image/webp/decode.go b/vendor/golang.org/x/image/webp/decode.go index 2371808f421..15dc0ee5561 100644 --- a/vendor/golang.org/x/image/webp/decode.go +++ b/vendor/golang.org/x/image/webp/decode.go @@ -134,6 +134,12 @@ func decode(r io.Reader, configOnly bool) (image.Image, image.Config, error) { wantAlpha = (buf[0] & alphaBit) != 0 widthMinusOne = uint32(buf[4]) | uint32(buf[5])<<8 | uint32(buf[6])<<16 heightMinusOne = uint32(buf[7]) | uint32(buf[8])<<8 | uint32(buf[9])<<16 + if uint64(widthMinusOne+1)*uint64(heightMinusOne+1) > 1<<32-1 { + // The product of _Canvas Width_ and _Canvas Height_ MUST be + // at most 2^32 - 1. + // https://www.rfc-editor.org/rfc/rfc9649.html#section-2.7-12 + return nil, image.Config{}, errInvalidFormat + } if configOnly { if wantAlpha { return nil, image.Config{ diff --git a/vendor/modules.txt b/vendor/modules.txt index 1767914b261..c103ca6b257 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2281,7 +2281,7 @@ golang.org/x/exp/slices golang.org/x/exp/slog golang.org/x/exp/slog/internal golang.org/x/exp/slog/internal/buffer -# golang.org/x/image v0.38.0 +# golang.org/x/image v0.39.0 ## explicit; go 1.25.0 golang.org/x/image/bmp golang.org/x/image/ccitt