Skip to content
Snippets Groups Projects
Verified Commit 55adbfcd authored by Volker Schukai's avatar Volker Schukai :alien:
Browse files

feat: implement jobqueue

parent 5f8be4ff
No related branches found
No related tags found
No related merge requests found
.envrc 0 → 100644
source_url "https://raw.githubusercontent.com/cachix/devenv/d1f7b48e35e6dee421cfd0f51481d17f77586997/direnvrc" "sha256-YBzqskFZxmNb3kYVoKD9ZixoPXJh1C9ZvTLGFRkauZ0="
use devenv
\ No newline at end of file
# Devenv
.devenv*
devenv.local.nix
# direnv
.direnv
# pre-commit
.pre-commit-config.yaml
smell.go
# THIS FILE IS AUTOGENERATED BY THE DEVENVSHELL
# DO NOT EDIT THIS FILE MANUALLY
# INSTEAD EDIT THE DEVENVSHELL CONFIGURATION FILE devenv.nix
# AND OPEN A SHELL WITH THE COMMAND devenv shell
#
image: docker-registry.schukai.com:443/nixos-ci-devenv:latest
services:
- docker:dind
variables:
# The repo name as used in
# https://github.com/nix-community/NUR/blob/master/repos.json
NIXOS_VERSION: "23.05"
NIXPKGS_ALLOW_UNFREE: "1"
NIXPKGS_ALLOW_INSECURE: "1"
DOCKER_DRIVER: overlay2
GIT_DEPTH: 10
stages:
- test
- deploy
before_script:
- nix shell nixpkgs#coreutils-full -c mkdir -p /certs/client/
- nix shell nixpkgs#coreutils-full -c ln -fs /etc/ssl/certs/ca-bundle.crt /certs/client/ca.pem
- echo > .env-gitlab-ci
- variables=("HOME=$HOME" "CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME" "CI_REPOSITORY_URL=$CI_REPOSITORY_URL" "GITLAB_TOKEN=$GITLAB_TOKEN" "CI_JOB_TOKEN=$CI_JOB_TOKEN" "GITLAB_USER_EMAIL=$GITLAB_USER_EMAIL" "GITLAB_USER_NAME=\"$GITLAB_USER_NAME\"" "CI_REGISTRY_USER=$CI_REGISTRY_USER" "CI_PROJECT_ID=$CI_PROJECT_ID" "CI_PROJECT_DIR=$CI_PROJECT_DIR" "CI_API_V4_URL=$CI_API_V4_URL" "CI_PROJECT_NAME=$CI_PROJECT_NAME" "CI_COMMIT_SHORT_SHA=$CI_COMMIT_SHORT_SHA"); for var in "${variables[@]}"; do echo "$var" >> .env-gitlab-ci; done
- cat .env-gitlab-ci
after_script:
- if [ -f .env-gitlab-ci ]; then rm .env-gitlab-ci; fi
test:
stage: test
tags:
- nixos
script:
- devenv shell test-lib
cache:
- key: nixos
paths:
- /nix/store
artifacts:
paths:
- dist
deploy:
stage: deploy
tags:
- nixos
script:
- devenv shell -c deploy-lib
when: on_success
cache:
- key: nixos
paths:
- /nix/store
artifacts:
paths:
- dist
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="accountSettings">
<option name="activeProfile" value="profile:default" />
<option name="activeRegion" value="eu-west-1" />
<option name="recentlyUsedProfiles">
<list>
<option value="profile:default" />
</list>
</option>
<option name="recentlyUsedRegions">
<list>
<option value="eu-west-1" />
</list>
</option>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MarkdownSettings">
<enabledExtensions>
<entry key="MermaidLanguageExtension" value="false" />
<entry key="PlantUMLLanguageExtension" value="true" />
</enabledExtensions>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/job-queues.iml" filepath="$PROJECT_DIR$/.idea/job-queues.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>
\ No newline at end of file
LICENSE 0 → 100644
Copyright (C) 2022 by schukai GmbH.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
# Job Queues # Job Queues
The jobque allows the processing of tasks. The `jobQueue` library in Go aims to serve as a Cron replacement, enabling the scheduled and event-driven execution of tasks in an organized manner.
\ No newline at end of file
## Library Requirements
### Core Requirements
1. **Job Management Structure**: A structure to catalog all jobs, providing methods to add and remove jobs.
2. **YAML Import**: A function to import job definitions from a YAML file.
3. **Job Exclusivity**: Jobs should be either concurrently runnable or exclusive.
4. **System Resource Monitoring**: Real-time monitoring of system resources like CPU, memory, etc., consumed by the jobs.
5. **Priority and Distribution**: Each job should have a priority. Jobs should also be evenly distributed across available resources.
6. **Single and Recurring Jobs**: Support for both one-time and recurring jobs.
7. **Logging**: Each job should have a log written that includes Process ID, memory usage, start time, end time, and exit code.
8. **Priority Escalation**: If a job is not run due to its priority, its priority should escalate over time.
### Additional Requirements
1. **Error Handling**: Mechanisms for effective error handling, especially for failed or terminated jobs.
2. **Notification System**: Optionally, an interface for notifications on job failures or completions.
3. **Resource Limits**: Ability to set resource limits per job.
4. **Job Dependencies**: Optional support for jobs that depend on the successful completion of other jobs.
5. **Testability**: The library should be easy to test, ideally via unit tests.
6. **Documentation**: Comprehensive API documentation and a guide on how the library works.
## Documentation
### `JobData` Structure
#### Fields
- **Id (JobIDType)**
- Unique identifier for the job.
- **Priority (int)**
- The priority level of the job in the queue.
- **Exclusive (bool)**
- Specifies whether the job should be executed exclusively or not.
- **MaxRuns (int)**
- The maximum number of times the job should be executed.
- **Concurrency (int)**
- The maximum number of concurrent executions of the job.
- **LastRun (time.Time)**
- Timestamp for when the job was last executed.
- **NextRun (time.Time)**
- Timestamp for the next scheduled execution of the job.
- **Logs ([]JobLog)**
- An array of log entries for the job.
- **Schedule (string)**
- A cron expression that defines the execution schedule.
- **Status (JobStatus)**
- The current status of the job (e.g., `Running`, `Completed`).
- **Timeout (time.Duration)**
- The maximum duration the job is allowed to run.
- **Retries (int)**
- The number of times the job will be retried if it fails.
- **RetryDelay (time.Duration)**
- The delay before retrying the job if it fails.
- **ResourceLimits (struct)**
- Resource limits for CPU and Memory.
- **CPULimit (float64)**
- CPU limit for the job.
- **MemoryLimit (uint64)**
- Memory limit for the job.
- **Dependencies ([]JobIDType)**
- List of job IDs this job depends on.
- **Tags ([]string)**
- Tags associated with the job for easier categorization.
- **Metadata (map[string]interface{})**
- Metadata for extending the structure with additional information.
- **Stats (JobStats)**
- Job-related statistics.
### `Job` Structure
Inherits fields from `JobData` and adds additional functional fields.
#### Fields
- **Runnable (Runnable)**
- A function or method that defines the action of the job. This is what gets executed.
- **scheduleImpl (cron.Schedule)**
- Internal cron schedule implementation that interprets the `Schedule` string in `JobData`.
- **TelemetryHooks ([]func(*JobLog))**
- An array of functions that will be called for telemetry during the job's execution. These hooks can update a `JobLog` object.
- **Failover (func() error)**
- A function that gets called if the primary execution fails for some reason. It's meant for fail over mechanisms.
- **ctx (context.Context)**
- A context that can carry deadlines, cancellations, and other request-scoped values across API boundaries and between processes.
- **mu (sync.Mutex)**
- Mutex for synchronizing access to the job's fields, making the job safe for concurrent use.
# THIS FILE IS AUTOGENERATED BY THE DEVENVSHELL
# DO NOT EDIT THIS FILE MANUALLY
# INSTEAD EDIT THE DEVENVSHELL CONFIGURATION FILE devenv.nix
# AND OPEN A SHELL WITH THE COMMAND devenv shell
#
# Information about the task runner can be found here:
# https://taskfile.dev
version: '3'
tasks:
default:
cmds:
- task --list
silent: true
test:
desc: Execute unit tests in Go.
cmds:
- echo "Execute unit tests in Go."
- go test -cover -v ./...
- go test -bench -v ./...
- go test -race -v ./...
test-fuzz:
desc: Conduct fuzzing tests.#
cmds:
- echo "Conduct fuzzing tests."
- go test -v -fuzztime=30s -fuzz=Fuzz ./...
add-licenses:
desc: Attach license headers to Go files.
cmds:
- echo "Attach license headers to Go files."
- go install github.com/google/addlicense@latest
- addlicense -c "schukai GmbH" -s -l "AGPL-3.0" ./*.go
silent: true
check-licenses:
desc: Check license headers of Go files.
silent: true
cmds:
- go-licenses save "$(get-go-default-packages)" --ignore "gitlab.schukai.com" --force --save_path ${DEVENV_ROOT}/licenses/
check:
desc: Confirm repository status.
cmds:
- git diff-index --quiet HEAD || (echo "There are uncommitted changes after running make. Please commit or stash them before running make."; exit 1)
silent: true
commit:
desc: Commit changes to the repository.
aliases:
- c
- ci
- git-commit
cmds:
- do-git-commit
package main
import (
"fmt"
jobqueue "gitlab.schukai.com/oss/libraries/go/services/job-queues.git"
)
func main() {
jq := jobqueue.NewJobs()
if jq == nil {
panic("NewJobs returned nil")
}
// Hinzufügen eines neuen Jobs
err := jq.AddJob(jobqueue.JobSpecification{
Id: "test",
Priority: 1,
}, &jobqueue.ExternalProcessRunner{
Command: "sleep",
Args: []string{"1"},
})
if err != nil {
panic(err)
}
// Abrufen aller Jobs
allJobs := jq.GetJobs()
fmt.Println("Alle Jobs:", allJobs)
// Entfernen eines Jobs
removed, err := jq.RemoveJob("test")
if err != nil {
panic(err)
}
if removed {
fmt.Println("Job wurde entfernt.")
}
}
{
"nodes": {
"devenv": {
"locked": {
"dir": "src/modules",
"lastModified": 1696344641,
"narHash": "sha256-cfGsdtDvzYaFA7oGWSgcd1yST6LFwvjMcHvtVj56VcU=",
"owner": "cachix",
"repo": "devenv",
"rev": "05e26941f34486bff6ebeb4b9c169b6f637f1758",
"type": "github"
},
"original": {
"dir": "src/modules",
"owner": "cachix",
"repo": "devenv",
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1685518550,
"narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
"pre-commit-hooks",
"nixpkgs"
]
},
"locked": {
"lastModified": 1660459072,
"narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1696374741,
"narHash": "sha256-gt8B3G0ryizT9HSB4cCO8QoxdbsHnrQH+/BdKxOwqF0=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "8a4c17493e5c39769f79117937c79e1c88de6729",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-23.05",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1685801374,
"narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "c37ca420157f4abc31e26f436c1145f8951ff373",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.05",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1696374741,
"narHash": "sha256-gt8B3G0ryizT9HSB4cCO8QoxdbsHnrQH+/BdKxOwqF0=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "8a4c17493e5c39769f79117937c79e1c88de6729",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-23.05",
"type": "indirect"
}
},
"pre-commit-hooks": {
"inputs": {
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"gitignore": "gitignore",
"nixpkgs": [
"nixpkgs"
],
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1696516544,
"narHash": "sha256-8rKE8Je6twTNFRTGF63P9mE3lZIq917RAicdc4XJO80=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "66c352d33e0907239e4a69416334f64af2c685cc",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"type": "github"
}
},
"root": {
"inputs": {
"devenv": "devenv",
"nixpkgs": "nixpkgs",
"pre-commit-hooks": "pre-commit-hooks",
"version": "version"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"version": {
"inputs": {
"nixpkgs": "nixpkgs_2"
},
"locked": {
"lastModified": 1690668568,
"narHash": "sha256-jzixQKFFW4oxO0S4GYqbkFCXzhBd6com6Z9+MtVKakU=",
"ref": "refs/heads/master",
"rev": "3838f03165b726e47d586c04a1821749375e1001",
"revCount": 37,
"type": "git",
"url": "https://gitlab.schukai.com/oss/utilities/version.git"
},
"original": {
"type": "git",
"url": "https://gitlab.schukai.com/oss/utilities/version.git"
}
}
},
"root": "root",
"version": 7
}
{ pkgs, inputs, phps, lib, config, modulesPath, ... }:
{
# https://devenv.sh/packages/
packages = with pkgs; [
inputs.version.defaultPackage."${builtins.currentSystem}"
appimage-run
blackbox
blackbox-terminal
coreutils-full
dbeaver
delve
dialog
drill
exa
fd
fd
gcc12
gdlv
git
glab
gnugrep
gnumake
gnused
go-licenses
go-task
gum
httpie
hurl
jq
libffi
logrotate
meld
memcached
netcat
nixfmt
procps
ranger
unixtools.xxd
unzip
util-linux
wget
zlib
nodePackages.mermaid-cli
feh
];
# https://devenv.sh/languages/
# languages.nix.enable = true;
languages = { go = { enable = true; }; };
difftastic.enable = true;
scripts.draw-graph.exec = ''
echo -e "Enter Meirmaid graph definition. ''${RED}End with Ctrl+D''${RESET}\n"
diagram=$(${pkgs.gum}/bin/gum write --placeholder "Enter Meirmaid graph definition. End with Ctrl+D")
tmpOutput=$(mktemp).png
echo "$diagram" | ${pkgs.nodePackages.mermaid-cli}/bin/mmdc -i - -o "$tmpOutput"
${pkgs.feh}/bin/feh $tmpOutput
# should delte the file, but does not work ask with gum
${pkgs.gum}/bin/gum confirm "Delete temporary file?"
if [ $? -eq 0 ]; then
rm "$tmpOutput"
else
echo "not deleting; file is at $tmpOutput"
fi
'';
scripts.get-go-default-packages.exec = ''
#!${pkgs.bash}/bin/bash
echo $(awk -F ' ' '/^module / { print $2 }' go.mod)
'';
# This script is executed when the app is built
# You can use it to build the app
scripts.test-lib.exec = ''
#!${pkgs.bash}/bin/bash
#set -euo pipefail
set -x
PATH="''${PATH}":${pkgs.coreutils}/bin
PATH="''${PATH}":${pkgs.findutils}/bin
PATH="''${PATH}":${pkgs.jq}/bin/
PATH="''${PATH}":${pkgs.rsync}/bin/
PATH="''${PATH}":${pkgs.bash}/bin/
PATH="''${PATH}":${pkgs.curl}/bin/
PATH="''${PATH}":${pkgs.moreutils}/bin/
PATH="''${PATH}":${pkgs.gnutar}/bin
PATH="''${PATH}":${pkgs.gzip}/bin/
PATH="''${PATH}":${pkgs.procps}/bin/
PATH="''${PATH}":${pkgs.exa}/bin/
PATH="''${PATH}":${pkgs.git}/bin/
PATH="''${PATH}":${pkgs.gnugrep}/bin/
PATH="''${PATH}":${
inputs.version.defaultPackage."${builtins.currentSystem}"
}/bin/
export PATH
task test
'';
# This scritp is used to deploy the app to the gitlab registry
# It is used by the gitlab-ci.yml file
# The environment variables are set in the gitlab project settings
scripts.deploy-lib.exec = ''
#!${pkgs.bash}/bin/bash
PATH="''${PATH}":${pkgs.coreutils}/bin
PATH="''${PATH}":${pkgs.jq}/bin/
PATH="''${PATH}":${pkgs.curl}/bin/
PATH="''${PATH}":${pkgs.moreutils}/bin/
PATH="''${PATH}":${pkgs.gnutar}/bin
PATH="''${PATH}":${pkgs.gzip}/bin/
PATH="''${PATH}":${pkgs.exa}/bin/
PATH="''${PATH}":${pkgs.git}/bin/
PATH="''${PATH}":${
inputs.version.defaultPackage."${builtins.currentSystem}"
}/bin/
export PATH
if [[ -f .env-gitlab-ci ]]; then
source .env-gitlab-ci
rm .env-gitlab-ci
fi
set -x
## if $HOME not set, set it to current directory
if [[ -z "''${HOME}" ]]; then
HOME=$(pwd)
fi
export HOME
git config user.email "''${GITLAB_USER_EMAIL}"
git config user.name "''${GITLAB_USER_NAME:-"Gitlab CI"}"
git config pull.rebase true
git config http.sslVerify "false"
git remote set-url origin https://pad:''${GITLAB_TOKEN}@''${CI_REPOSITORY_URL#*@}
git fetch --all --tags --unshallow
git reset --hard origin/master
git checkout $CI_COMMIT_REF_NAME
git pull origin $CI_COMMIT_REF_NAME
if [ ! -z "''${CI_PROJECT_DIR}" ]; then
echo "CI_PROJECT_DIR is set, using it as project root."
project_root=$(realpath "''${CI_PROJECT_DIR}")/
elif [ ! -z "''${DEVENV_ROOT}" ]; then
echo "DEVENV_ROOT is set, using it as project root."
project_root=$(realpath "''${DEVENV_ROOT}")/
else
echo "Error: DEVENV_ROOT or CI_PROJECT_DIR environment variables are not set."
exit 1
fi
if [ ! -d "''${project_root}" ]; then
echo "Error: Project root directory does not seem to be valid."
echo "Check the DEVENV_ROOT or CI_PROJECT_DIR environment variables."
exit 1
fi
if [ -z "'CI_JOB_TOKEN" ]; then
echo "Error: CI_JOB_TOKEN variable is not set."
exit 1
fi
git --no-pager log --decorate=short --pretty=oneline
gitVersion=v$(version predict)
git tag -a $gitVersion -m"chore: bump version"
git --no-pager log --decorate=short --pretty=oneline
git push -o ci.skip origin ''${CI_COMMIT_REF_NAME} --tags
echo "done"
'';
enterShell = ''
cat <<'EOF' > Taskfile.yml
# THIS FILE IS AUTOGENERATED BY THE DEVENVSHELL
# DO NOT EDIT THIS FILE MANUALLY
# INSTEAD EDIT THE DEVENVSHELL CONFIGURATION FILE devenv.nix
# AND OPEN A SHELL WITH THE COMMAND devenv shell
#
# Information about the task runner can be found here:
# https://taskfile.dev
version: '3'
tasks:
default:
cmds:
- task --list
silent: true
test:
desc: Execute unit tests in Go.
cmds:
- echo "Execute unit tests in Go."
- go test -cover -v ./...
- go test -bench -v ./...
- go test -race -v ./...
test-fuzz:
desc: Conduct fuzzing tests.#
cmds:
- echo "Conduct fuzzing tests."
- go test -v -fuzztime=30s -fuzz=Fuzz ./...
add-licenses:
desc: Attach license headers to Go files.
cmds:
- echo "Attach license headers to Go files."
- go install github.com/google/addlicense@latest
- addlicense -c "schukai GmbH" -s -l "AGPL-3.0" ./*.go
silent: true
check-licenses:
desc: Check license headers of Go files.
silent: true
cmds:
- go-licenses save "$(get-go-default-packages)" --ignore "gitlab.schukai.com" --force --save_path ''${DEVENV_ROOT}/licenses/
check:
desc: Confirm repository status.
cmds:
- git diff-index --quiet HEAD || (echo "There are uncommitted changes after running make. Please commit or stash them before running make."; exit 1)
silent: true
commit:
desc: Commit changes to the repository.
aliases:
- c
- ci
- git-commit
cmds:
- do-git-commit
EOF
cat <<'EOF' > .gitlab-ci.yml
# THIS FILE IS AUTOGENERATED BY THE DEVENVSHELL
# DO NOT EDIT THIS FILE MANUALLY
# INSTEAD EDIT THE DEVENVSHELL CONFIGURATION FILE devenv.nix
# AND OPEN A SHELL WITH THE COMMAND devenv shell
#
image: docker-registry.schukai.com:443/nixos-ci-devenv:latest
services:
- docker:dind
variables:
# The repo name as used in
# https://github.com/nix-community/NUR/blob/master/repos.json
NIXOS_VERSION: "23.05"
NIXPKGS_ALLOW_UNFREE: "1"
NIXPKGS_ALLOW_INSECURE: "1"
DOCKER_DRIVER: overlay2
GIT_DEPTH: 10
stages:
- test
- deploy
before_script:
- nix shell nixpkgs#coreutils-full -c mkdir -p /certs/client/
- nix shell nixpkgs#coreutils-full -c ln -fs /etc/ssl/certs/ca-bundle.crt /certs/client/ca.pem
- echo > .env-gitlab-ci
- variables=("HOME=$HOME" "CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME" "CI_REPOSITORY_URL=$CI_REPOSITORY_URL" "GITLAB_TOKEN=$GITLAB_TOKEN" "CI_JOB_TOKEN=$CI_JOB_TOKEN" "GITLAB_USER_EMAIL=$GITLAB_USER_EMAIL" "GITLAB_USER_NAME=\"$GITLAB_USER_NAME\"" "CI_REGISTRY_USER=$CI_REGISTRY_USER" "CI_PROJECT_ID=$CI_PROJECT_ID" "CI_PROJECT_DIR=$CI_PROJECT_DIR" "CI_API_V4_URL=$CI_API_V4_URL" "CI_PROJECT_NAME=$CI_PROJECT_NAME" "CI_COMMIT_SHORT_SHA=$CI_COMMIT_SHORT_SHA"); for var in "''${variables[@]}"; do echo "$var" >> .env-gitlab-ci; done
- cat .env-gitlab-ci
after_script:
- if [ -f .env-gitlab-ci ]; then rm .env-gitlab-ci; fi
test:
stage: test
tags:
- nixos
script:
- devenv shell test-lib
cache:
- key: nixos
paths:
- /nix/store
artifacts:
paths:
- dist
deploy:
stage: deploy
tags:
- nixos
script:
- devenv shell -c deploy-lib
when: on_success
cache:
- key: nixos
paths:
- /nix/store
artifacts:
paths:
- dist
EOF
'';
scripts.do-git-commit.exec = ''
#!/usr/bin/env bash
# Define colors if the terminal supports it
if [ -t 1 ]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
RESET='\033[0m'
BOLD='\033[1m'
else
RED=""
GREEN=""
RESET=""
fi
step=1
reset
clear
# create random log file
LOGFILE="$(mktemp)"
if [ $? -ne 0 ]; then
echo -e "''${RED}✖ Could not create temporary log file. Exiting.''${RESET}"
exit 1
fi
log_and_display() {
echo -e "''${GREEN}==> $step. $1''${RESET}" | tee -a $LOGFILE
step=$((step + 1))
}
log_error_and_display() {
echo -e "''${RED}==> $step. $1''${RESET}" | tee -a $LOGFILE
}
printLogfileAndExit() {
exit_code=$1
echo -e "\n\n========================================\n\n\n"
echo -e "\n\n''${BOLD}Git and GitLab Automation Script''${RESET}\n\nI have performed the following actions:\n\n"
cat "$LOGFILE"
# Optional: Remove log file
rm -f "$LOGFILE"
if [ $exit_code -eq 0 ]; then
echo -e "\n''${GREEN}✔''${RESET} All actions were successful" | tee -a $LOGFILE
elif [ $exit_code -eq -1 ]; then
echo -e "\n''${RED}✖''${RESET} The script was manually cancelled" | tee -a $LOGFILE
exit_code=0
else
echo -e "\n''${RED}✖''${RESET} Some actions failed" | tee -a $LOGFILE
fi
exit $exit_code
}
print_headline() {
local title=$1
local underline=$(printf '─%.0s' $(seq 1 ''${#title}))
echo -e "\n\n''${BOLD}''${title}\n''${underline}''${RESET}\n"
}
do_cancel() {
echo -e "''${RED}==> ✖ Cancelled.''${RESET}" | tee -a $LOGFILE
printLogfileAndExit -1
}
# Function for unified logging and display
log_action() {
if [ $? -eq 0 ]; then
echo -e " ''${GREEN}✔''${RESET} $1: Successful" | tee -a $LOGFILE
else
echo -e " ''${RED}✖''${RESET} $1: Failed" | tee -a $LOGFILE
printLogfileAndExit 1
fi
}
print_what_to_do() {
echo -e "\n\nWhat do you want to do?\n"
}
git_status=$(git status --porcelain)
if [[ -z "$git_status" ]]; then
log_error_and_display "No changes to commit. Exiting."
printLogfileAndExit 0
fi
print_headline "Choose commit type"
selection=$(gum choose "feat: (new feature for the user, not a new feature for build script)" "fix: (bug fix for the user, not a fix to a build script)" "chore: (updating grunt tasks etc.; no production code change)" "docs: (changes to the documentation)" "style: (formatting, missing semi colons, etc; no production code change)" "refactor: (refactoring production code, eg. renaming a variable)" "test: (adding missing tests, refactoring tests; no production code change)" "Cancel")
commit_type=$(echo "$selection" | awk -F':' '{print $1}')
if [[ "$commit_type" == "Cancel" ]]; then
do_cancel
fi
log_and_display "You chose the commit type: $commit_type"
# NEXT STEP ISSUE HANDLING ############################################################################################################
#log_and_display "Issue handling"
gitlabIssues=()
while IFS= read -r line; do
if [[ $line =~ ^# ]]; then
id=$(echo "$line" | awk '{print substr($1, 2)}')
title=$(echo "$line" | awk -F'about' '{print $1}' | awk '{$1=$2=""; print substr($0, 3)}')
gitlabIssues+=("$id > $title")
fi
done < <(gum spin --spinner dot --show-output --title "Ask gitlab ..." -- glab issue list --output-format=details)
## if issues are available, ask if user wants to use an existing issue or create a new one
createOption="Create new issue"
existingOption="Use existing issue"
cancelOption="Cancel"
print_headline "Choose issue handling"
if [ ''${#gitlabIssues[@]} -eq 0 ]; then
log_and_display "There are no issues available."
print_what_to_do
choice=$(gum choose "$createOption" "$cancelOption")
else
log_and_display "There are ''${#gitlabIssues[@]} issues available."
print_what_to_do
choice=$(gum choose "$createOption" "$existingOption" "$cancelOption")
fi
if [[ "$choice" == "$cancelOption" ]]; then
do_cancel
fi
## array of issue ids
work_on_issue_ids=()
issue_text=""
if [[ "$choice" == "$createOption" ]]; then
print_headline "Create new issue"
issue_text=$(gum input --placeholder "Enter issue title")
echo -e "Enter issue description. ''${RED}End with Ctrl+D''${RESET}\n"
issue_description=$(gum write --placeholder "Enter issue description. End with Ctrl+D")
if [[ -z "$issue_text" ]]; then
log_error_and_display "Issue title is empty. Exiting."
printLogfileAndExit 1
fi
log_and_display "You entered the issue title: $issue_text"
log_and_display "You entered the issue description: $issue_description"
echo -e "\n"
gum confirm "Do you want to create this issue?"
# gum confirm exits with status 0 if confirmed and status 1 if cancelled.
if [ $? -eq 1 ]; then
do_cancel
fi
issue_output=$(glab issue create -t"$issue_text" --no-editor --description "$issue_description")
issue_id=$(echo "$issue_output" | grep -oP '(?<=/issues/)\d+')
work_on_issue_ids+=("$issue_id")
log_action "glab issue with id $issue_id created"
else
print_headline "Use existing issue"
echo -e "Select issue with arrow keys and press tab or space to select. Press enter to confirm.\n"
issue_ids=$(gum choose --no-limit "''${gitlabIssues[@]}")
# assign issue_ids to work_on_issue_ids. iterate over lines and take integer from beginning of line
while IFS= read -r line; do
work_on_issue_ids+=($(echo "$line" | grep -oP '^\d+'))
done <<<"$issue_ids"
fi
if [ ''${#work_on_issue_ids[@]} -eq 0 ]; then
log_and_display "No issue selected. Exiting."
printLogfileAndExit 0
fi
# NEXT STEP COMMIT MESSAGE ############################################################################################################
# print work_on_issue_ids
work_on_issue_ids_string=""
for i in "''${work_on_issue_ids[@]}"; do
work_on_issue_ids_string+="#$i "
done
log_and_display "You chose to work on the following issues: ''${work_on_issue_ids_string}"
print_headline "Check for changes to commit"
# ' ' = unmodified
# M = modified
# T = file type changed (regular file, symbolic link or submodule)
# A = added
# D = deleted
# R = renamed
# C = copied (if config option status.renames is set to "copies")
# U = updated but unmerged
# https://man.freebsd.org/cgi/man.cgi?query=git-status&sektion=1&manpath=freebsd-release-ports
count_all_changes=$(echo "$git_status" | wc -l)
count_staged_changes=$(echo "$git_status" | grep -c '^M')
count_new_staged_files=$(echo "$git_status" | grep -c '^A')
count_staged_changes=$((count_staged_changes + count_new_staged_files))
git_options_all="All $count_all_changes changes"
git_options_staged="Only the $count_staged_changes staged changes"
git_options_select_files="Select files"
git_options_cancel="Cancel"
git_options_array=()
if [[ $count_all_changes -gt 0 ]]; then
git_options_array+=("$git_options_all")
fi
if [[ $count_staged_changes -gt 0 ]]; then
git_options_array+=("$git_options_staged")
fi
git_options_array+=( "$git_options_select_files" )
git_options_array+=( "$git_options_cancel" )
selection=$(gum choose "''${git_options_array[@]}")
if [[ "$selection" == "$git_options_cancel" ]]; then
do_cancel
fi
if [[ "$selection" == "$git_options_all" ]]; then
git add -A
echo "1"
elif [[ "$selection" == "$git_options_select_files" ]]; then
files=()
while IFS= read -r line; do
files+=("$line")
done <<<"$git_status"
selected_files=$(gum choose --no-limit "''${files[@]}")
# no files selected
if [[ -z "$selected_files" ]]; then
log_and_display "No files selected. Exiting."
printLogfileAndExit 0
fi
# add selected files
while IFS= read -r line; do
## git proclimne could have letter, ? or ! at the beginning of the line
file=$(echo "$line" | awk '{print $2}')
if [[ -z "$file" || ! -f "$file" ]]; then
log_and_display "No file found in line: $line"
continue
fi
git add "$file"
done <<<"$selected_files"
fi
## count staged changes again and print
count_staged_changes=$(echo "$git_status" | grep -c '^M')
count_new_staged_files=$(echo "$git_status" | grep -c '^A')
count_staged_changes=$((count_staged_changes + count_new_staged_files))
log_and_display "You have $count_staged_changes staged changes to commit."
# NEXT STEP COMMIT MESSAGE ############################################################################################################
print_headline "Enter commit message"
commit_message=$(gum input --placeholder "Enter commit message" --value "$commit_type: $issue_text $work_on_issue_ids_string")
if [[ -z "$commit_message" ]]; then
log_error_and_display "Commit message is empty. Exiting."
printLogfileAndExit 1
fi
log_and_display "You entered the commit message: $commit_message"
gum confirm "Do you want to commit with this message?"
if [ $? -eq 1 ]; then
do_cancel
fi
# NEXT STEP COMMIT ####################################################################################################################
print_headline "Committing changes"
if ! git commit -m "$commit_message" ; then
log_error_and_display "Commit failed. Exiting."
printLogfileAndExit 1
fi
log_and_display "Commit successful."
# NEXT STEP PUSH ######################################################################################################################
print_headline "Pushing changes"
if ! git push ; then
log_error_and_display "Push failed. Exiting."
printLogfileAndExit 1
fi
log_and_display "Push successful."
# Close issue ######################################################################################################################
print_headline "Closing issues"
for issue_id in "''${work_on_issue_ids[@]}"; do
gum confirm "Do you want to close issue #$issue_id?"
if [ $? -eq 1 ]; then
continue
fi
if ! glab issue close "$issue_id" ; then
log_error_and_display "Closing issue $issue_id failed. Exiting."
else
log_and_display "Closing issue $issue_id successful."
fi
done
printLogfileAndExit 0
'';
}
inputs:
nixpkgs:
url: github:nixos/nixpkgs/nixos-23.05
version:
url: git+https://gitlab.schukai.com/oss/utilities/version.git
flake: true
error.go 0 → 100644
package jobqueue
import "fmt"
var (
ErrCPUPercentage = fmt.Errorf("could not get CPU percentage")
ErrIntervalIsZero = fmt.Errorf("interval must not be 0")
ErrResourceLimitExceeded = fmt.Errorf("resource limit exceeded")
ErrNoRunDefined = fmt.Errorf("no runnable function defined")
ErrCycleDetected = fmt.Errorf("cycle detected")
ErrJobAlreadyExists = fmt.Errorf("job already exists")
ErrUnknownDependency = fmt.Errorf("unknown dependency")
ErrMissingDependency = fmt.Errorf("missing dependency")
ErrJobNotFound = fmt.Errorf("job not found")
ErrJobIsDependency = fmt.Errorf("job is a dependency")
ErrNotRunning = fmt.Errorf("job queue is not running")
ErrAlreadyPaused = fmt.Errorf("job queue is already paused")
ErrAlreadyStopped = fmt.Errorf("job queue is already stopped")
ErrNotPaused = fmt.Errorf("job queue is not paused")
ErrAlreadyStarted = fmt.Errorf("job queue is already started")
ErrTimeout = fmt.Errorf("job timed out")
//ErrInitializationFailed = fmt.Errorf("resource monitoring initialization failed")
)
package jobqueue
import (
"context"
"sync"
"sync/atomic"
"time"
)
const (
MODE_STOPPED = iota
MODE_RUNNING
MODE_PAUSED
)
type mode int32
type shouldStopFunc func(executor *jobExecutor) bool
type jobExecutor struct {
mutex sync.Mutex
Queue JobsInterface
Ctx context.Context
CancelFunc context.CancelFunc
MaxParallel int
interval time.Duration
Ticker *time.Ticker
cleanupTimer *time.Ticker
//stopChan chan struct{}
sem chan struct{}
pauseChan chan struct{}
resumeChan chan struct{}
runningMode int32
doneChan chan struct{}
shouldStop shouldStopFunc
}
func NewJobExecutor(queue JobsInterface, maxParallel int, interval time.Duration, shouldStopFunc shouldStopFunc) *jobExecutor {
execCtx, cancelFunc := context.WithCancel(context.Background())
return &jobExecutor{
Queue: queue,
Ctx: execCtx,
CancelFunc: cancelFunc,
MaxParallel: maxParallel,
interval: interval,
Ticker: nil,
cleanupTimer: nil,
//stopChan: make(chan struct{}),
sem: make(chan struct{}, maxParallel),
pauseChan: make(chan struct{}),
resumeChan: make(chan struct{}),
runningMode: MODE_STOPPED,
shouldStop: shouldStopFunc,
}
}
func (je *jobExecutor) executeJobs() {
je.mutex.Lock()
je.Ticker = time.NewTicker(je.interval)
je.cleanupTimer = time.NewTicker(10 * je.interval)
je.setRunningFlag(MODE_RUNNING)
je.mutex.Unlock()
for {
select {
case <-je.Ticker.C:
if !je.IsPaused() {
je.runJobs()
if je.shouldStop != nil && je.shouldStop(je) {
_ = je.Stop()
return
}
}
case <-je.cleanupTimer.C:
je.Queue.Cleanup()
case <-je.pauseChan:
je.setRunningFlag(MODE_PAUSED)
case <-je.resumeChan:
je.setRunningFlag(MODE_RUNNING)
case <-je.Ctx.Done():
je.mutex.Lock()
je.Ticker.Stop()
je.cleanupTimer.Stop()
je.setRunningFlag(MODE_STOPPED)
je.mutex.Unlock()
return
}
}
}
func (je *jobExecutor) runJobs() {
// Get jobs that can be executed
jobs := je.Queue.GetExecutableJobs()
if len(jobs) == 0 {
return
}
var wg sync.WaitGroup
// Map to track the status of executed jobs
jobStatus := make(map[JobIDType]bool)
for _, job := range jobs {
jobStatus[job.GetId()] = false
}
// Channel for coordinating job execution
jobChan := make(chan ReadOnlyJob)
// Determine the number of jobs to be sent to the channel
for _, job := range jobs {
dependencies := job.GetDependencies()
canRun := true
for _, dependencyID := range dependencies {
// Check if dependencies have already been executed
if !jobStatus[dependencyID] {
canRun = false
break
}
}
if canRun {
wg.Add(1) // Increment the WaitGroup counter for each job to be sent
}
}
// Loop through all jobs and execute only if dependencies are met
go func() {
for _, job := range jobs {
dependencies := job.GetDependencies()
canRun := true
for _, dependencyID := range dependencies {
// Check if dependencies have already been executed
if !jobStatus[dependencyID] {
canRun = false
break
}
}
if canRun {
// Send the job to the job channel
jobChan <- job
}
}
close(jobChan) // Close the channel after all jobs have been sent
}()
maxParallel := je.MaxParallel
if len(jobs) < maxParallel {
maxParallel = len(jobs)
}
// Execute jobs in parallel
for i := 0; i < maxParallel; i++ {
go func() {
for job := range jobChan {
job.Run(je.Ctx)
// Mark the job as executed
jobStatus[job.GetId()] = true
wg.Done()
}
}()
}
// Wait for all jobs to complete before returning
wg.Wait()
}
func (je *jobExecutor) Start() error {
if je.IsRunning() {
return ErrAlreadyStarted
}
if je.IsPaused() {
return je.Resume()
}
go je.executeJobs()
return nil
}
func (je *jobExecutor) Stop() error {
if !je.IsRunning() && !je.IsPaused() {
return ErrAlreadyStopped
}
je.CancelFunc()
return nil
}
func (je *jobExecutor) Pause() error {
if je.IsPaused() {
return ErrAlreadyPaused
}
je.mutex.Lock()
je.pauseChan <- struct{}{}
je.setRunningFlag(MODE_PAUSED)
je.mutex.Unlock()
return nil
}
func (je *jobExecutor) Resume() error {
if !je.IsPaused() {
return ErrNotPaused
}
je.mutex.Lock()
je.resumeChan <- struct{}{}
je.setRunningFlag(MODE_RUNNING)
je.mutex.Unlock()
return nil
}
func (je *jobExecutor) IsPaused() bool {
return atomic.LoadInt32(&je.runningMode) == MODE_PAUSED
}
func (je *jobExecutor) IsRunning() bool {
return atomic.LoadInt32(&je.runningMode) == MODE_RUNNING
}
func (je *jobExecutor) setRunningFlag(mode int32) {
atomic.StoreInt32(&je.runningMode, mode)
}
package jobqueue
import (
"context"
"testing"
"time"
)
type TestRunnable struct{}
func (r TestRunnable) Run(ctx context.Context) (int, any, error) {
// Dummy run implementation
return 0, nil, nil
}
func TestJobExecutorStartAndStop(t *testing.T) {
queue := NewJobs()
executor := NewJobExecutor(queue, 1, time.Millisecond*50, nil)
// Fügen Sie einen Job zur Warteschlange hinzu
err := queue.AddJob(JobSpecification{
Id: "test-job",
Priority: 1,
Concurrency: 1,
}, TestRunnable{})
if err != nil {
t.Fatalf("Failed to add job: %v", err)
}
err = executor.Start()
if err != nil {
t.Errorf("Failed to start executor: %v", err)
}
for i := 0; i < 10; i++ {
time.Sleep(time.Millisecond * 100)
if executor.IsRunning() {
break
}
}
if !executor.IsRunning() {
t.Errorf("Executor should be running")
}
err = executor.Stop()
for i := 0; i < 10; i++ {
time.Sleep(time.Millisecond * 100)
if !executor.IsRunning() {
break
}
}
if err != nil {
t.Errorf("Failed to stop executor: %v", err)
}
if executor.IsRunning() {
t.Errorf("Executor should not be running")
}
}
func TestJobExecutorPauseAndResume(t *testing.T) {
queue := NewJobs()
executor := NewJobExecutor(queue, 1, time.Millisecond*50, nil)
// Fügen Sie einen Job zur Warteschlange hinzu
err := queue.AddJob(JobSpecification{
Id: "test-job",
Priority: 1,
Concurrency: 1,
}, TestRunnable{})
if err != nil {
t.Fatalf("Failed to add job: %v", err)
}
err = executor.Start()
if err != nil {
t.Errorf("Failed to start executor: %v", err)
}
for i := 0; i < 10; i++ {
time.Sleep(time.Millisecond * 100)
if executor.IsRunning() {
break
}
}
err = executor.Pause()
if err != nil {
t.Errorf("Failed to pause executor: %v", err)
}
for i := 0; i < 10; i++ {
time.Sleep(time.Millisecond * 100)
if executor.IsPaused() {
break
}
}
if !executor.IsPaused() {
t.Errorf("Executor should be paused")
}
err = executor.Resume()
if err != nil {
t.Errorf("Failed to resume executor: %v", err)
}
for i := 0; i < 10; i++ {
time.Sleep(time.Millisecond * 100)
if executor.IsRunning() {
break
}
}
if executor.IsPaused() {
t.Errorf("Executor should not be paused")
}
err = executor.Stop()
if err != nil {
t.Errorf("Failed to stop executor: %v", err)
}
for i := 0; i < 10; i++ {
time.Sleep(time.Millisecond * 100)
if !executor.IsRunning() {
break
}
}
if executor.IsRunning() {
t.Errorf("Executor should not be running")
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment