Repository navigation
Expand file tree
/
Copy pathMakefile
More file actions
256 lines (197 loc) · 10.9 KB
/
Copy pathMakefile
File metadata and controls
256 lines (197 loc) · 10.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
# The commands used in this Makefile expect to be interpreted by bash.
SHELL := /bin/bash
TESTS=$(shell go list ./... | grep -v /vendor/ | grep -v github-release-notes)
git_commit := $(shell git rev-parse --short HEAD)
git_branch := $(shell git symbolic-ref -q --short HEAD)
git_upstream := $(shell git remote get-url $(shell git config branch.$(shell git symbolic-ref -q --short HEAD).remote) 2> /dev/null)
export GIT_BRANCH = $(git_branch)
export BRANCH ?= $(shell git symbolic-ref -q --short HEAD 2>/dev/null || git rev-parse --short HEAD)
export GIT_UPSTREAM = $(git_upstream)
FUNNEL_VERSION ?= $(shell version=$$(git describe --tags --exact-match 2>/dev/null || git rev-parse --short HEAD 2>/dev/null || printf unknown); if git rev-parse --verify HEAD >/dev/null 2>&1 && ! git diff-index --quiet HEAD --; then version=$${version}-dirty; fi; printf '%s' "$$version")
export FUNNEL_VERSION
GORELEASER_VERSION := v2.12.0
GORELEASER := $(CURDIR)/build/tools/goreleaser
VERSION_LDFLAGS=\
-X "github.com/calypr/funnel/version.BuildDate=$(shell date)" \
-X "github.com/calypr/funnel/version.GitCommit=$(git_commit)" \
-X "github.com/calypr/funnel/version.GitBranch=$(git_branch)" \
-X "github.com/calypr/funnel/version.GitUpstream=$(git_upstream)" \
-X "github.com/calypr/funnel/version.Version=$(FUNNEL_VERSION)"
export CGO_ENABLED=0
# Build the code
install:
@touch version/version.go
@go install -ldflags '$(VERSION_LDFLAGS)' .
# Build the code
build:
@touch version/version.go
@go build -ldflags '$(VERSION_LDFLAGS)' -buildvcs=false .
# Build an unoptimized version of the code for use during debugging
# https://go.dev/doc/gdb
debug:
@go install -gcflags=all="-N -l"
@funnel server run
# Generate the protobuf/gRPC code using Buf
proto: proto-depends
@go run ./api/tes/internal/openapi2proto ./api/tes/task-execution-schemas/openapi/task_execution_service.openapi.yaml > api/tes/tes.proto
@go run ./api/tes/internal/openapi2proto -service ./api/tes/task-execution-schemas/openapi/task_execution_service.openapi.yaml > api/internal/tesrpc/service.proto
@cd api && PATH="$(shell go env GOPATH)/bin:$$PATH" buf generate
@mv api/internal/tesrpc/service.swagger.json api/tes/tes.swagger.json
# Initialize the TES schema submodule and install Buf.
# `proto` depends on this target so generation also works from a fresh clone.
proto-depends:
@test -f api/tes/task-execution-schemas/openapi/task_execution_service.openapi.yaml || git submodule update --init --recursive api/tes/task-execution-schemas
@GOTOOLCHAIN=$(shell go env GOVERSION) go install github.com/bufbuild/buf/cmd/buf@v1.28.1
@GOTOOLCHAIN=$(shell go env GOVERSION) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.12
@GOTOOLCHAIN=$(shell go env GOVERSION) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0
@GOTOOLCHAIN=$(shell go env GOVERSION) go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@v2.26.3
@GOTOOLCHAIN=$(shell go env GOVERSION) go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@v2.26.3
# Lint Protobuf files
proto-lint:
@cd api && buf lint
# Start API reference doc server
serve-doc:
@go get golang.org/x/tools/cmd/godoc
godoc --http=:6060
# Automatically update code formatting
tidy:
@go get golang.org/x/tools/cmd/goimports
@find . \( -path ./vendor -o -path ./webdash/node_modules -o -path ./venv -o -path ./.git \) -prune -o -type f -print | grep -v "\.pb\." | grep -E '.*\.go$$' | xargs goimports -w
@find . \( -path ./vendor -o -path ./webdash/node_modules -o -path ./venv -o -path ./.git \) -prune -o -type f -print | grep -v "\.pb\." | grep -E '.*\.go$$' | xargs gofmt -w -s
lint-depends:
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.14.0
go install golang.org/x/tools/cmd/goimports
# Run code style and other checks
lint:
@golangci-lint run
.PHONY: deadcode deadcode-update
deadcode:
@python3 .github/scripts/check-deadcode.py
deadcode-update:
@python3 .github/scripts/check-deadcode.py --update-baseline
# Run all tests
test:
@go test $(TESTS)
test-verbose:
@go test -v $(TESTS)
start-mongodb:
@docker rm -f funnel-mongodb-test > /dev/null 2>&1 || echo
@docker run -d --name funnel-mongodb-test -p 27000:27017 docker.io/mongo > /dev/null
test-mongodb:
@go version
@go test ./tests/integration/tasks/ --funnel-config `pwd`/tests/testdata/config/mongo.config.yml
@go test ./tests/integration/scheduler/ --funnel-config `pwd`/tests/testdata/config/mongo.config.yml
start-postgres:
@docker rm -f funnel-postgres-test > /dev/null 2>&1 || echo
@docker run -d --name funnel-postgres-test -p 5432:5432 \
-e POSTGRES_PASSWORD=postgres \
docker.io/postgres > /dev/null
test-postgres:
@go version
@go test ./tests/integration/tasks/ --funnel-config `pwd`/tests/testdata/config/postgres.config.yml
@go test ./tests/integration/scheduler/ --funnel-config `pwd`/tests/testdata/config/postgres.config.yml
test-badger:
@go version
@go test ./tests/integration/tasks/ -funnel-config `pwd`/tests/testdata/config/badger.config.yml
start-dynamodb:
@docker rm -f funnel-dynamodb-test > /dev/null 2>&1 || echo
@docker run -d --name funnel-dynamodb-test -p 18000:8000 docker.io/amazon/dynamodb-local > /dev/null
test-dynamodb:
@go test ./tests/integration/tasks/ -funnel-config `pwd`/tests/testdata/config/dynamo.config.yml
start-datastore:
@docker rm -f funnel-datastore-test > /dev/null 2>&1 || echo
@docker run -d --name funnel-datastore-test -p 12432:12432 google/cloud-sdk:latest gcloud beta emulators datastore start --no-store-on-disk --host-port 0.0.0.0:12432 --project funnel-test
test-datastore:
DATASTORE_EMULATOR_HOST=localhost:12432 \
go test ./tests/integration/tasks/ -funnel-config `pwd`/tests/testdata/config/datastore.config.yml
start-kafka:
@docker rm -f funnel-kafka > /dev/null 2>&1 || echo
@docker run -d --name funnel-kafka -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST="localhost" --env ADVERTISED_PORT=9092 spotify/kafka
test-kafka:
@go test ./tests/integration/events/kafka/ -funnel-config `pwd`/tests/testdata/config/kafka.config.yml
test-htcondor:
@docker pull ohsucompbio/htcondor
@go test -timeout 120s ./tests/integration/compute/htcondor -funnel-config `pwd`/tests/testdata/config/htcondor.config.yml
test-slurm:
@go version
@docker pull quay.io/ohsu-comp-bio/slurm
@go test -timeout 120s ./tests/integration/compute/slurm -funnel-config `pwd`/tests/testdata/config/slurm.config.yml
test-gridengine:
@docker pull ohsucompbio/gridengine
@go test -timeout 120s ./tests/integration/compute/gridengine -funnel-config `pwd`/tests/testdata/config/gridengine.config.yml
test-pbs-torque:
@docker pull ohsucompbio/pbs-torque
@go test -timeout 120s ./tests/integration/compute/pbs -funnel-config `pwd`/tests/testdata/config/pbs.config.yml
test-amazon-s3:
@go test -v ./tests/integration/storage -funnel-config `pwd`/tests/testdata/config/s3.config.yml -run TestAmazonS3
start-generic-s3:
@docker rm -f funnel-s3server > /dev/null 2>&1 || echo
@docker run -d --name funnel-s3server -p 18888:8000 -e REMOTE_MANAGEMENT_DISABLE=1 zenko/cloudserver
@docker rm -f funnel-minio > /dev/null 2>&1 || echo
@docker run -d --name funnel-minio -p 9000:9000 -e "MINIO_ROOT_USER=fakekey" -e "MINIO_ROOT_PASSWORD=fakesecret" -e "MINIO_REGION=us-east-1" quay.io/cdis/minio:chainguard-latest-2026-09-25@sha256:a05a4497e8dce3cb7a7a1bf1872ba5d30ea988f1e8c22c9e0920503761c4b5f1 server /data
test-generic-s3:
@go test -v ./tests/integration/storage -funnel-config `pwd`/tests/testdata/config/amazoncli-minio-s3.config.yml -run TestAmazonS3Storage
@go test -v ./tests/integration/storage -funnel-config `pwd`/tests/testdata/config/scality-s3.config.yml -run TestGenericS3Storage
@go test -v ./tests/integration/storage -funnel-config `pwd`/tests/testdata/config/minio-s3.config.yml -run TestGenericS3Storage
@go test -v ./tests/integration/storage -funnel-config `pwd`/tests/testdata/config/multi-s3.config.yml -run TestGenericS3Storage
test-gs:
@go test ./tests/integration/storage -run TestGoogleStorage -funnel-config `pwd`/tests/testdata/config/gs.config.yml ${GCE_PROJECT_ID}
test-swift:
@go test ./tests/integration/storage -funnel-config `pwd`/tests/testdata/config/swift.config.yml -run TestSwiftStorage
start-pubsub:
@docker rm -f funnel-pubsub-test > /dev/null 2>&1 || echo
@docker run -d --name funnel-pubsub-test -p 8085:8085 google/cloud-sdk:latest gcloud beta emulators pubsub start --project funnel-test --host-port 0.0.0.0:8085
test-pubsub:
@PUBSUB_EMULATOR_HOST=localhost:8085 \
go test ./tests/integration/events/pubsub/ -funnel-config `pwd`/tests/testdata/config/pubsub.config.yml
start-ftp:
@cd tests/integration/storage/ftp-server/ && ./start-server.sh
test-ftp:
@go test -v ./tests/integration/storage -run TestFTPStorage -funnel-config `pwd`/tests/testdata/config/ftp.config.yml
# Build the web dashboard (assets in internal/webdash/build are embedded into the
# binary via go:embed)
webdash:
@cd webdash && npm run build
# Build binaries for all OS/Architectures
snapshot: release-dep
@"$(GORELEASER)" release --clean --snapshot
# Build a docker container locally
docker:
docker build -t quay.io/ohsu-comp-bio/funnel:latest ./
# Create a release on GitHub using the reviewed changelog section.
.PHONY: release release-dry-run release-notes release-dep snapshot
release-notes:
@mkdir -p build
@release_ref="$${GORELEASER_CURRENT_TAG:-$$(git rev-parse HEAD)}"; \
awk -v ref="$$release_ref" '/^# Unreleased$$/ { printing=1; next } printing && /^# / { exit } printing { gsub(/\]\(docs\//, "](https://github.com/calypr/funnel/blob/" ref "/docs/"); print }' CHANGELOG.md > build/release-notes.md
@test -s build/release-notes.md
release: release-dep release-notes
@"$(GORELEASER)" release --clean --release-notes build/release-notes.md
release-dry-run: release-dep release-notes
@"$(GORELEASER)" release --clean --release-notes build/release-notes.md --skip=publish,announce
# Install dependencies for release
release-dep:
@mkdir -p "$(CURDIR)/build/tools"
@GOBIN="$(CURDIR)/build/tools" go install github.com/goreleaser/goreleaser/v2@$(GORELEASER_VERSION)
# Generate mocks for testing.
gen-mocks: gen-mocks-deps
@mockery --dir internal/compute/scheduler --name Client --inpackage --output internal/compute/scheduler
@mockery --dir internal/compute/scheduler --name SchedulerServiceServer --inpackage --output internal/compute/scheduler
gen-mocks-deps:
@go install github.com/vektra/mockery/v2@v2.20.0
# Make everything usually needed to prepare for a pull request
full: proto install tidy lint test website webdash
hugo-deps:
@cd website && go mod download
# Build the website
website: hugo-deps
@python3 .github/scripts/check-docs.py
@hugo --source ./website --minify
@npx -y pagefind@1.4.0 --site build/website
# Serve the Funnel website on http://localhost:1313
website-dev: website
@hugo --source ./website --watch server --baseURL http://localhost:1313/
# Remove build/development files.
clean:
@rm -rf ./bin ./pkg ./test_tmp ./build ./buildtools
.PHONY: proto proto-depends proto-lint website docker webdash build debug