diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ceb7c42a..f9cb4de8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,7 +39,7 @@ security-scan: APPSECHUB_PARENT_PIPELINE_ID: $CI_PIPELINE_ID APPSECHUB_SCA_SBOM_GENERATOR: custom APPSECHUB_SBOM_PATH: sbom.cyclonedx.json - APPSECHUB_SBOM_MASK: "*bom*.json" + APPSECHUB_SBOM_MASK: '*bom*.json' CUSTOM_SBOM_GENERATOR_JOB_NAME: sbom-creation rules: - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH diff --git a/mddocs/en/Makefile b/mddocs/en/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/mddocs/en/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/mddocs/en/_static/custom.css b/mddocs/en/_static/custom.css new file mode 100644 index 00000000..ac17ca7e --- /dev/null +++ b/mddocs/en/_static/custom.css @@ -0,0 +1,3 @@ +.logo { + width: 200px !important; +} diff --git a/mddocs/en/_static/icon.svg b/mddocs/en/_static/icon.svg new file mode 100644 index 00000000..186f363e --- /dev/null +++ b/mddocs/en/_static/icon.svg @@ -0,0 +1,146 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mddocs/en/_static/logo.svg b/mddocs/en/_static/logo.svg new file mode 100644 index 00000000..ce803900 --- /dev/null +++ b/mddocs/en/_static/logo.svg @@ -0,0 +1,2731 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mddocs/en/_static/logo_no_title.svg b/mddocs/en/_static/logo_no_title.svg new file mode 100644 index 00000000..e2021113 --- /dev/null +++ b/mddocs/en/_static/logo_no_title.svg @@ -0,0 +1,2693 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mddocs/en/_static/metrics.prom b/mddocs/en/_static/metrics.prom new file mode 100644 index 00000000..0a831744 --- /dev/null +++ b/mddocs/en/_static/metrics.prom @@ -0,0 +1 @@ +# Generated in CI diff --git a/mddocs/en/_static/openapi.json b/mddocs/en/_static/openapi.json new file mode 100644 index 00000000..7b3bf552 --- /dev/null +++ b/mddocs/en/_static/openapi.json @@ -0,0 +1,9 @@ +{ + "openapi": "3.1.0", + "version": "unknown", + "info": { + "title": "Generated in CI", + "version": "unknown" + }, + "paths": {} +} \ No newline at end of file diff --git a/mddocs/en/_static/redoc.html b/mddocs/en/_static/redoc.html new file mode 100644 index 00000000..3c4246e5 --- /dev/null +++ b/mddocs/en/_static/redoc.html @@ -0,0 +1,28 @@ + + + + + Horizon - ReDoc + + + + + + + + + + + + + + diff --git a/mddocs/en/_static/stats.prom b/mddocs/en/_static/stats.prom new file mode 100644 index 00000000..0a831744 --- /dev/null +++ b/mddocs/en/_static/stats.prom @@ -0,0 +1 @@ +# Generated in CI diff --git a/mddocs/en/_static/swagger.html b/mddocs/en/_static/swagger.html new file mode 100644 index 00000000..0f891cd9 --- /dev/null +++ b/mddocs/en/_static/swagger.html @@ -0,0 +1,26 @@ + + + + + + + SwaggerUI + + + + +
+ + + + diff --git a/mddocs/en/backend/architecture.md b/mddocs/en/backend/architecture.md new file mode 100644 index 00000000..06597cb7 --- /dev/null +++ b/mddocs/en/backend/architecture.md @@ -0,0 +1,30 @@ +# Architecture { #backend-architecture } + +```plantuml + + @startuml + title Backend artitecture + skinparam linetype polyline + left to right direction + + actor "User" + + frame "Horizon" { + component "REST API" + database "Database" + } + + component "LDAP" + + [User] --> [REST API] + [REST API] --> [Database] + [REST API] ..> [LDAP] + @enduml +``` + +```mermaid +stateDiagram-v2 +[User] --> [RESTAPI] +[RESTAPI] --> [Database] +[RESTAPI] --> [LDAP] +``` diff --git a/mddocs/en/backend/auth/cached_ldap.md b/mddocs/en/backend/auth/cached_ldap.md new file mode 100644 index 00000000..92407673 --- /dev/null +++ b/mddocs/en/backend/auth/cached_ldap.md @@ -0,0 +1,172 @@ +# LDAP Cached Auth provider { #backend-auth-ldap-cached } + +## Description { #cached_ldap-description } + +Same as [LDAP Auth provider][backend-auth-ldap-cached], but if LDAP request for checking user credentials was successful, +credentials are stored in local cache (table in internal database, in form `login` + `hash(password)` + `update timestamp`). + +Next auth requests for the same login are performed against this cache **first**. LDAP requests are send *only* if cache have been expired. + +This allows to: + +- Bypass errors with LDAP availability, e.g. network errors +- Reduce number of requests made to LDAP. + +Downsides: + +- If user changed password, and cache is not expired yet, user may still log in with old credentials. +- Same if user was blocked in LDAP. + +## Interaction schema { #cached_ldap-interaction-schema } + +```plantuml + + @startuml + title CachedLDAPAuthProvider + participant "Client" + participant "Backend" + participant "LDAP" + + == POST v1/auth/token == + + activate "Client" + alt First time auth | Empty cache | Cache expired + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : Search for credentials cache by login + "Backend" --> "Backend" : No items found or item expired, using LDAP + "Backend" --> "Backend" : DN = bind_dn_template(login) + "Backend" -> "LDAP" ++ : Call bind(DN, password) + "LDAP" --[#green]> "Backend" -- : Successful + "Backend" --> "Backend" : Check user in internal backend database,\nusername = login + "Backend" -> "Backend" : Create user if not exist + "Backend" -> "Backend" : Save credentials to cache + "Backend" -[#green]> "Client" -- : Generate and return access_token + + else Using cache, LDAP is totally ignored + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : Search for credentials cache by login + "Backend" --> "Backend" : Found credentials, check for expiration + "Backend" --> "Backend" : Not expired, validate password is matching hash + "Backend" --> "Backend" : Password match, not calling LDAP + "Backend" --> "Backend" : Check user in internal backend database + "Backend" -> "Backend" : Create user if not exist + "Backend" -[#green]> "Client" -- : Generate and return access_token + + else Password mismatch with cache, LDAP is totally ignored + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : Search for credentials cache by login + "Backend" --> "Backend" : Found credentials, check for expiration + "Backend" --> "Backend" : Not expired, validate password is matching hash + "Backend" --> "Backend" : Password do not match local cache + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else No cache or cache expired, LDAP is unavailable + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : Search for credentials cache by login + "Backend" --> "Backend" : No items found or item expired, using LDAP + "Backend" --> "Backend" : DN = bind_dn_template(login) + "Backend" -[#red]>x "LDAP" : Call bind(DN, password) + "Backend" x-[#red]> "Client" -- : 503 Service unavailable + + else + note right of "Client" : Other cases are same as for LDAPAuthProvider,\nlike lookup, blocked/deleted users + end + + == GET v1/namespaces == + + alt Successful case + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" -> "Backend" : Get data + "Backend" -[#green]> "Client" -- : Return data + + else Token is expired + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is blocked + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is deleted + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 404 Not found + end + + deactivate "Client" + @enduml +``` + + +## Configuration { #cached_ldap-configuration } + +Other settings are just the same as for `LDAPAuthProvider` + +::: horizon.backend.settings.auth.cached_ldap.CachedLDAPAuthProviderSettings + + +::: horizon.backend.settings.auth.cached_ldap.LDAPCacheSettings + +::: horizon.backend.settings.auth.cached_ldap.LDAPCachePasswordHashSettings diff --git a/mddocs/en/backend/auth/custom.md b/mddocs/en/backend/auth/custom.md new file mode 100644 index 00000000..37f38c0c --- /dev/null +++ b/mddocs/en/backend/auth/custom.md @@ -0,0 +1,5 @@ +# Custom Auth provider { #backend-auth-custom } + +You can implement custom auth provider by inheriting from class below and implementing necessary methods. + +::: horizon.backend.providers.auth.AuthProvider diff --git a/mddocs/en/backend/auth/dummy.md b/mddocs/en/backend/auth/dummy.md new file mode 100644 index 00000000..1279ab2c --- /dev/null +++ b/mddocs/en/backend/auth/dummy.md @@ -0,0 +1,114 @@ +# Dummy Auth provider { #backend-auth-dummy } + +## Description { #dummy-description } + +This auth provider allows to sign-in with any username and password, and and then issues an access token. + +After successful auth, username is saved to backend database. It is then used for creating audit records for any object change, see `changed_by` field. + +## Interaction schema { #dummy-interaction-schema } + +```plantuml + @startuml + title DummyAuthProvider + participant "Client" + participant "Backend" + + == POST v1/auth/token == + + activate "Client" + alt Successful case + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : Password is completely ignored + "Backend" --> "Backend" : Check user in internal backend database + "Backend" -> "Backend" : Create user if not exist + "Backend" -[#green]> "Client" -- : Generate and return access_token + + else User is blocked + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : Password is completely ignored + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is deleted + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : Password is completely ignored + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 404 Not found + end + + == GET v1/namespaces == + + alt Successful case + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" -> "Backend" : Get data + "Backend" -[#green]> "Client" -- : Return data + + else Token is expired + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is blocked + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is deleted + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 404 Not found + end + + deactivate "Client" + @enduml +``` + + +## Configuration { #dummy-configuration } + +::: horizon.backend.settings.auth.dummy.DummyAuthProviderSettings + +::: horizon.backend.settings.auth.jwt.JWTSettings diff --git a/mddocs/en/backend/auth/index.md b/mddocs/en/backend/auth/index.md new file mode 100644 index 00000000..4c0526e3 --- /dev/null +++ b/mddocs/en/backend/auth/index.md @@ -0,0 +1,26 @@ +# Auth Providers { #backend-auth-providers } + +Horizon supports different auth provider implementations. You can change implementation via settings: + +::: horizon.backend.settings.auth.AuthSettings + +## Auth providers + +* [Dummy Auth provider][backend-auth-dummy] + * [Description][dummy-description] + * [Interaction schema][dummy-interaction-schema] + * [Configuration][dummy-configuration] +* [LDAP Auth provider][backend-auth-ldap] + * [Description][ldap-description] + * [Strategies][ldap-strategies] + * [Interaction schema][ldap-interaction-schema] + * [Basic configuration][ldap-basic-configuration] + * [Lookup-related configuration][ldap-lookup-related-configuration] +* [LDAP Cached Auth provider][backend-auth-ldap-cached] + * [Description][cached_ldap-description] + * [Interaction schema][cached_ldap-interaction-schema] + * [Configuration][cached_ldap-configuration] + +## For developers + +* [Custom Auth provider][backend-auth-custom] diff --git a/mddocs/en/backend/auth/ldap.md b/mddocs/en/backend/auth/ldap.md new file mode 100644 index 00000000..e40f6c34 --- /dev/null +++ b/mddocs/en/backend/auth/ldap.md @@ -0,0 +1,333 @@ +# LDAP Auth provider { #backend-auth-ldap } + +## Description { #ldap-description } + +This auth provider checks for user credentials in LDAP, and and then issues an access token. + +All requests to backend should be made with passing this access token. If token is expired, then new auth token should be issued. + +After successful auth, username is saved to backend database. It is then used for creating audit records for any object change, see `changed_by` field. + +### WARNING + +Until token is valid, no requests will be made to LDAP to check if user exists and not locked. +So do not set access token expiration time for too long (e.g. longer than a day). + +## Strategies { #ldap-strategies } + +### NOTE + +Basic LDAP terminology is explained here: [LDAP Overview](https://www.zytrax.com/books/ldap/ch2/) + +There are 2 strategies to check for user in LDAP: + +- Try to call `bind` request in LDAP with `DN` (`DistinguishedName`) and user password. `DN` is generated using [bind_dn_template][horizon.backend.settings.auth.ldap.LDAPSettings.bind_dn_template] +- First try to *lookup* for user (`search` request) in LDAP to get user's `DN` using some query, and then try to call `bind` using this `DN`. See [lookup settings][horizon.backend.settings.auth.ldap.LDAPSettings.lookup] + +By default, **lookup strategy is used**, as it can find user in a complex LDAP/ActiveDirectory environment. For example: + +- you can search for user by `uid`, e.g. `(uid={login})` or `(sAMAccountName={login})` +- you can search for user by several attributes, e.g. `(|(uid={login})(mail={login}@domain.com))` +- you can filter for entries, like `(&(uid={login})(objectClass=person)` +- you can filter for users matching a specific group or some other condition, like `(&(uid={login})(memberOf=cn=MyPrettyGroup,ou=Groups,dc=mycompany,dc=com))` + +After user is found in LDAP, its [uid_attribute][horizon.backend.settings.auth.ldap.LDAPSettings.uid_attribute] is used for audit records. + +## Interaction schema { #ldap-interaction-schema } + +```plantuml + + @startuml + title LDAPAuthProvider (no lookup) + participant "Client" + participant "Backend" + participant "LDAP" + + == POST v1/auth/token == + + activate "Client" + alt Successful case + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : DN = bind_dn_template(login) + "Backend" -> "LDAP" ++ : Call bind(DN, password) + "LDAP" --[#green]> "Backend" -- : Successful + "Backend" --> "Backend" : Check user in internal backend database,\nusername = login + "Backend" -> "Backend" : Create user if not exist + "Backend" -[#green]> "Client" -- : Generate and return access_token + + else Wrong credentials | User blocker in LDAP + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : DN = bind_dn_template(login) + "Backend" -> "LDAP" ++ : Call bind(DN, password) + "LDAP" x-[#red]> "Backend" -- : Bind error + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is blocked in internal backend database + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : DN = bind_dn_template(login) + "Backend" -> "LDAP" ++ : Call bind(DN, password) + "LDAP" --[#green]> "Backend" -- : Successful + "Backend" --> "Backend" : Check user in internal backend database,\nusername = login + "Backend" x-[#red]> "Client" -- : 404 Not found + + else User is deleted in internal backend database + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : DN = bind_dn_template(login) + "Backend" -> "LDAP" ++ : Call bind(DN, password) + "LDAP" --[#green]> "Backend" -- : Return user info + "Backend" --> "Backend" : Check user in internal backend database,\nusername = login + "Backend" x-[#red]> "Client" -- : 404 Not found + + else LDAP is unavailable + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : DN = bind_dn_template(login) + "Backend" -[#red]>x "LDAP" : Call bind(DN, password) + "Backend" x-[#red]> "Client" -- : 503 Service unavailable + end + + == GET v1/namespaces == + + alt Successful case + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" -> "Backend" : Get data + "Backend" -[#green]> "Client" -- : Return data + + else Token is expired + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is blocked + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is deleted + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 404 Not found + end + + deactivate "Client" + @enduml +``` + +```mermaid +sequenceDiagram +participant "Client" +participant "Backend" +participant "LDAP" +activate "Client" +alt Successful case +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" ->> "LDAP" : Call bind(DN, password) +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = login +"Backend" ->> "Backend" : Create user if not exist +else +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" ->> "LDAP" : Call bind(DN, password) +else +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" ->> "LDAP" : Call bind(DN, password) +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = login +else +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" ->> "LDAP" : Call bind(DN, password) +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = login +else +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +end +alt Successful case +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" ->> "Backend" : Get data +else +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +else +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +else +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +end +deactivate "Client" +``` + +```plantuml + + @startuml + title LDAPAuthProvider (with lookup) + participant "Client" + participant "Backend" + participant "LDAP" + + == Backend start == + + "Backend" ->o "LDAP" ++ : bind(lookup.username, lookup.password) + note right of "LDAP" : Open connection \npool for\nsearch queries\n(optional, recommended) + + == POST v1/auth/token == + + activate "Client" + alt Successful case + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : query = query_template(login) + "Backend" ->o "LDAP" : Call search(query, base_dn, attributes=*) + "LDAP" --[#green]> "Backend" : Return user DN and uid_attribute + "Backend" -> "LDAP" ++ : Call bind(DN, password) + "LDAP" --[#green]> "Backend" -- : Successful + "Backend" --> "Backend" : Check user in internal backend database,\nusername = uid_attribute from LDAP response + "Backend" -> "Backend" : Create user if not exist + "Backend" -[#green]> "Client" -- : Generate and return access_token + + else Wrong credentials | User blocker in LDAP + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : query = query_template(login) + "Backend" ->o "LDAP" : Call search(query, base_dn, attributes=*) + "LDAP" --[#green]> "Backend" : Return user DN and uid_attribute + "Backend" -> "LDAP" ++ : Call bind(DN, password) + "LDAP" x--[#red]> "Backend" -- : Bind error + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is blocked in internal backend database + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : query = query_template(login) + "Backend" ->o "LDAP" : Call search(query, base_dn, attributes=*) + "LDAP" --[#green]> "Backend" : Return user DN and uid_attribute + "Backend" -> "LDAP" ++ : Call bind(DN, password) + "LDAP" --[#green]> "Backend" -- : Successful + "Backend" --> "Backend" : Check user in internal backend database,\nusername = uid_attribute from LDAP response + "Backend" x-[#red]> "Client" -- : 404 Not found + + else User is deleted in internal backend database + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : query = query_template(login) + "Backend" ->o "LDAP" : Call search(query, base_dn, attributes=*) + "LDAP" --[#green]> "Backend" : Return user DN and uid_attribute + "Backend" -> "LDAP" ++ : Call bind(DN, password) + "LDAP" --[#green]> "Backend" -- : Successful + "Backend" --> "Backend" : Check user in internal backend database,\nusername = uid_attribute from LDAP response + "Backend" x-[#red]> "Client" -- : 404 Not found + + else LDAP is unavailable + "Client" -> "Backend" ++ : login + password + "Backend" --> "Backend" : query = query_template(login) + "Backend" -[#red]>x "LDAP" : Call search(query, base_dn, attributes=*) + "Backend" x-[#red]> "Client" -- : 503 Service unavailable + end + + == GET v1/namespaces == + + alt Successful case + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" -> "Backend" : Get data + "Backend" -[#green]> "Client" -- : Return data + + else Token is expired + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is blocked + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 401 Unauthorized + + else User is deleted + "Client" -> "Backend" ++ : access_token + "Backend" --> "Backend" : Validate token + "Backend" --> "Backend" : Check user in internal backend database + "Backend" x-[#red]> "Client" -- : 404 Not found + end + + deactivate "Client" + @enduml +``` + +```mermaid +sequenceDiagram +participant "Client" +participant "Backend" +participant "LDAP" +"Backend" ->>o "LDAP" : bind(lookup.username, lookup.password) +Note right of "LDAP" : Open connection \npool for\nsearch queries\n(optional, recommended) +activate "Client" +alt Successful case +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +"Backend" ->>o "LDAP" : Call search(query, base_dn, attributes=*) +"Backend" ->> "LDAP" : Call bind(DN, password) +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = uid_attribute from LDAP response +"Backend" ->> "Backend" : Create user if not exist +else +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +"Backend" ->>o "LDAP" : Call search(query, base_dn, attributes=*) +"Backend" ->> "LDAP" : Call bind(DN, password) +else +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +"Backend" ->>o "LDAP" : Call search(query, base_dn, attributes=*) +"Backend" ->> "LDAP" : Call bind(DN, password) +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = uid_attribute from LDAP response +else +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +"Backend" ->>o "LDAP" : Call search(query, base_dn, attributes=*) +"Backend" ->> "LDAP" : Call bind(DN, password) +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = uid_attribute from LDAP response +else +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +end +alt Successful case +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" ->> "Backend" : Get data +else +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +else +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +else +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +end +deactivate "Client" +``` + +## Basic configuration { #ldap-basic-configuration } + +::: horizon.backend.settings.auth.ldap.LDAPAuthProviderSettings + +::: horizon.backend.settings.auth.ldap.LDAPSettings + +::: horizon.backend.settings.auth.jwt.JWTSettings + +::: horizon.backend.settings.auth.ldap.LDAPConnectionPoolSettings + +## Lookup-related configuration { #ldap-lookup-related-configuration } + +::: horizon.backend.settings.auth.ldap.LDAPLookupSettings + +::: horizon.backend.settings.auth.ldap.LDAPCredentials diff --git a/mddocs/en/backend/configuration/cors.md b/mddocs/en/backend/configuration/cors.md new file mode 100644 index 00000000..4c7fe860 --- /dev/null +++ b/mddocs/en/backend/configuration/cors.md @@ -0,0 +1,5 @@ +# CORS settings { #backend-configuration-cors } + +These settings used to control [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) options. + +::: horizon.backend.settings.server.cors.CORSSettings diff --git a/mddocs/en/backend/configuration/database.md b/mddocs/en/backend/configuration/database.md new file mode 100644 index 00000000..f77b8909 --- /dev/null +++ b/mddocs/en/backend/configuration/database.md @@ -0,0 +1,3 @@ +# Database settings { #backend-configuration-database } + +::: horizon.backend.settings.database.DatabaseSettings diff --git a/mddocs/en/backend/configuration/debug.md b/mddocs/en/backend/configuration/debug.md new file mode 100644 index 00000000..c44289ac --- /dev/null +++ b/mddocs/en/backend/configuration/debug.md @@ -0,0 +1,139 @@ +# Enabling debug { #backend-configuration-debug } + +## Return debug info in backend responses + +By default, server does not add error details to response bodies, +to avoid exposing instance-specific information to end users. + +You can change this by setting: + +```console +$ export HORIZON__SERVER__DEBUG=False +$ # start backend +$ curl -XPOST http://localhost:8000/failing/endpoint ... +{ + "error": { + "code": "unknown", + "message": "Got unhandled exception. Please contact support", + "details": null, + }, +} +``` + +```console +$ export HORIZON__SERVER__DEBUG=True +$ # start backend +$ curl -XPOST http://localhost:8000/failing/endpoint ... +Traceback (most recent call last): +File ".../uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi + result = await app( # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +File ".../site-packages/uvicorn/middleware/proxy_headers.py", line 84, in __call__ + return await self.app(scope, receive, send) +``` + +### WARNING + +This is only for development environment only. Do **NOT** use on production! + +## Print debug logs on backend + +See [Logging settings][backend-configuration-logging], but replace log level `INFO` with `DEBUG`. + +## Fill up `X-Request-ID` header on backend + +Server can add `X-Request-ID` header to responses, which allows to match request on client with backend response. + +This is done by `request_id` middleware, which is enabled by default and can configured as described below: + +::: horizon.backend.settings.server.request_id.RequestIDSettings + +## Print request ID to backend logs + +This is done by adding a specific filter to logging handler: + +### `logging.yml` + +```default +# development usage only +version: 1 +disable_existing_loggers: false + +filters: + # Add request ID as extra field named `correlation_id` to each log record. + # This is used in combination with settings.server.request_id.enabled=True + # See https://github.com/snok/asgi-correlation-id#configure-logging + correlation_id: + (): asgi_correlation_id.CorrelationIdFilter + uuid_length: 32 + default_value: '-' + +formatters: + plain: + (): logging.Formatter + # Add correlation_id to log records + fmt: '%(asctime)s.%(msecs)03d %(processName)s:%(process)d %(name)s:%(lineno)d [%(levelname)s] %(correlation_id)s %(message)s' + datefmt: '%Y-%m-%d %H:%M:%S' + +handlers: + main: + class: logging.StreamHandler + formatter: plain + filters: [correlation_id] + stream: ext://sys.stdout + +loggers: + '': + handlers: [main] + level: INFO + propagate: false + uvicorn: + handlers: [main] + level: INFO + propagate: false +``` + +Resulting logs look like: + +```text +2023-12-18 17:14:11.711 uvicorn.access:498 [INFO] 018c15e97a068ae09484f8c25e2799dd 127.0.0.1:34884 - "GET /monitoring/ping HTTP/1.1" 200 +``` + +## Use `X-Request-ID` header on client + +If client got `X-Request-ID` header from backend, it is printed to logs with `DEBUG` level: + +```pycon +>>> import logging +>>> logging.basicConfig(level=logging.DEBUG) +>>> client.ping() +DEBUG:urllib3.connectionpool:http://localhost:8000 "GET /monitoring/ping HTTP/1.1" 200 15 +DEBUG:horizon.client.base:Request ID: '018c15e97a068ae09484f8c25e2799dd' +``` + +Also, if backend response was not successful, `Request ID` is added to exception message: + +```pycon +>>> client.get_namespace("unknown") +requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http://localhost:8000/v1/namespaces/unknown +Request ID: '018c15eb80fa81a6b38c9eaa519cd322' +``` + +## Fill up `X-Application-Version` header on backend + +Server can add `X-Application-Version` header to responses, which allows to determine which version of backend is deployed. + +This is done by `application_version` middleware, which is enabled by default and can configured as described below: + +::: horizon.backend.settings.server.application_version.ApplicationVersionSettings + +## Use `X-Application-Version` header on client + +If client got `X-Application-Version` header from backend, it is compared with client version. + +If versions do not match, a warning is shown: + +```pycon +>>> client.ping() +UserWarning: Horizon client version '0.0.9' does not match backend version '1.0.0'. Please upgrade. +``` diff --git a/mddocs/en/backend/configuration/index.md b/mddocs/en/backend/configuration/index.md new file mode 100644 index 00000000..22a5a5a9 --- /dev/null +++ b/mddocs/en/backend/configuration/index.md @@ -0,0 +1,16 @@ +# Configuration { #backend-configuration } + +* [Database][backend-configuration-database] +* [Logging][backend-configuration-logging] +* [Monitoring][backend-configuration-monitoring] +* [CORS][backend-configuration-cors] +* [Static_files][backend-configuration-static-files] +* [Openapi][backend-configuration-openapi] +* [Debug][backend-configuration-debug] + +::: horizon.backend.settings + options: + members: + - Settings + - server + - ServerSettings diff --git a/mddocs/en/backend/configuration/logging.md b/mddocs/en/backend/configuration/logging.md new file mode 100644 index 00000000..25e12b0a --- /dev/null +++ b/mddocs/en/backend/configuration/logging.md @@ -0,0 +1,3 @@ +# Logging settings { #backend-configuration-logging } + +::: horizon.backend.settings.server.log.LoggingSettings diff --git a/mddocs/en/backend/configuration/monitoring.md b/mddocs/en/backend/configuration/monitoring.md new file mode 100644 index 00000000..246ec0ff --- /dev/null +++ b/mddocs/en/backend/configuration/monitoring.md @@ -0,0 +1,23 @@ +# Setup monitoring { #backend-configuration-monitoring } + +Backend provides 2 endpoints with Prometheus compatible metrics: + +- `GET /monitoring/metrics` - server metrics, like number of requests per path and response status, CPU and RAM usage, and so on. + +## Example metrics + +```default +# Generated in CI +``` + +- `GET /monitoring/stats` - usage statistics, like number of users, namespaces, HWMs. + +## Example stats + +```default +# Generated in CI +``` + +These endpoints are enabled and configured using settings below: + +::: horizon.backend.settings.server.monitoring.MonitoringSettings diff --git a/mddocs/en/backend/configuration/openapi.md b/mddocs/en/backend/configuration/openapi.md new file mode 100644 index 00000000..7c28b95a --- /dev/null +++ b/mddocs/en/backend/configuration/openapi.md @@ -0,0 +1,12 @@ +# OpenAPI settings { #backend-configuration-openapi } + +These settings used to control exposing OpenAPI.json and SwaggerUI/ReDoc endpoints. + +::: horizon.backend.settings.server.openapi + options: + members: + - OpenAPISettings + - SwaggerSettings + - RedocSettings + - LogoSettings + - FaviconSettings diff --git a/mddocs/en/backend/configuration/static_files.md b/mddocs/en/backend/configuration/static_files.md new file mode 100644 index 00000000..340cc9c6 --- /dev/null +++ b/mddocs/en/backend/configuration/static_files.md @@ -0,0 +1,5 @@ +# Serving static files { #backend-configuration-static-files } + +These settings used to control serving static files by backend. + +::: horizon.backend.settings.server.static_files.StaticFilesSettings diff --git a/mddocs/en/backend/install.md b/mddocs/en/backend/install.md new file mode 100644 index 00000000..dedeeb4d --- /dev/null +++ b/mddocs/en/backend/install.md @@ -0,0 +1,167 @@ +# Install & run backend { #backend-install } + +## With docker + +### Requirements + +- [Docker](https://docs.docker.com/engine/install/) +- [docker-compose](https://github.com/docker/compose/releases/) + +### Installation process + +Docker will download backend image of Horizon & Postgres, and run them. +Options can be set via `.env` file or `environment` section in `docker-compose.yml` + +### `docker-compose.yml` + +```default +services: + db: + image: postgres:17 + restart: unless-stopped + environment: + POSTGRES_DB: horizon + POSTGRES_USER: horizon + POSTGRES_PASSWORD: 123UsedForTestOnly + POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=C --lc-ctype=C + ports: + - 5432:5432 + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: pg_isready + start_period: 5s + interval: 30s + timeout: 5s + retries: 3 + + backend: + image: mtsrus/horizon-backend:${VERSION:-latest} + restart: unless-stopped + env_file: .env.docker + environment: + # list here usernames which should be assigned SUPERADMIN role on application start + HORIZON__ENTRYPOINT__ADMIN_USERS: admin + # PROMETHEUS_MULTIPROC_DIR is required for multiple workers, see: + # https://prometheus.github.io/client_python/multiprocess/ + PROMETHEUS_MULTIPROC_DIR: /tmp/prometheus-metrics + # tmpfs dir is cleaned up each container restart + tmpfs: + - /tmp/prometheus-metrics:mode=1777 + ports: + - 8000:8000 + depends_on: + db: + condition: service_healthy + +volumes: + postgres_data: +``` + +### `.env.docker` + +```default +# See Backend -> Configuration documentation +HORIZON__DATABASE__URL=postgresql+asyncpg://horizon:123UsedForTestOnly@db:5432/horizon +HORIZON__AUTH__PROVIDER=horizon.backend.providers.auth.dummy.DummyAuthProvider +HORIZON__AUTH__ACCESS_TOKEN__SECRET_KEY=234UsedForTestOnly +HORIZON__AUTH__LDAP__URL=ldap://ldap:389 +HORIZON__AUTH__LDAP__BASE_DN=ou=people,dc=ldapmock,dc=local +HORIZON__AUTH__LDAP__LOOKUP__CREDENTIALS__USER=uid=adminuser1,ou=people,dc=ldapmock,dc=local +HORIZON__AUTH__LDAP__LOOKUP__CREDENTIALS__PASSWORD=password +HORIZON__SERVER__DEBUG=true +HORIZON__SERVER__LOGGING__PRESET=colored +HORIZON_TEST_SERVER_URL=http://backend:8000 +HORIZON__SERVER__CORS__ENABLED=True +HORIZON__SERVER__CORS__ALLOW_ORIGINS=["http://localhost:3000"] +HORIZON__SERVER__CORS__ALLOW_CREDENTIALS=True +HORIZON__SERVER__CORS__ALLOW_METHODS=["*"] +HORIZON__SERVER__CORS__ALLOW_HEADERS=["*"] +HORIZON__SERVER__CORS__EXPOSE_HEADERS=["X-Request-ID","Location","Access-Control-Allow-Credentials"] +``` + +After container is started and ready, open [http://localhost:8000/docs](http://localhost:8000/docs). + +Users listed in `HORIZON__ENTRYPOINT__ADMIN_USERS` env variable will be automatically promoted to `SUPERADMIN` role. + +## Without docker + +### Requirements without docker + +- Python 3.7 or above +- Pydantic 2.x +- `libldap2-dev`, `libsasl2-dev`, `libkrb5-dev` (for [LDAP Auth provider][backend-auth-ldap]) +- Some relation database instance, like [Postgres](https://www.postgresql.org/) + +### Installation process without docker + +Install `data-horizon` package with following *extra* dependencies: + +```console +$ pip install data-horizon[backend,postgres,ldap] +... +``` + +Available *extras* are: + +- `backend` - main backend requirements, like FastAPI, SQLAlchemy and so on. +- `postgres` - requirements required to use Postgres as backend data storage. +- `ldap` - requirements used by [LDAP Auth provider][backend-auth-ldap]. + +#### NOTE + +For **macOS** users, an additional step is required. [You need to install the “bonsai” Python library from source code](https://bonsai.readthedocs.io/en/latest/install.html#install-from-source-on-macos). This installation is necessary to work with LDAP. + +### Run database + +Start Postgres instance somewhere, and set up database url using environment variables: + +```bash +HORIZON__DATABASE__URL=postgresql+asyncpg://user:password@postgres-host:5432/database_name +``` + +You can use virtually any database supported by [SQLAlchemy](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls), +but the only one we really tested is Postgres. + +See [Database settings][backend-configuration-database] for more options. + +### Run migrations + +To apply migrations (database structure changes) you need to execute following command: + +```console +$ python -m horizon.backend.db.migrations upgrade head +... +``` + +This is a thin wrapper around [alembic](https://alembic.sqlalchemy.org/en/latest/tutorial.html#running-our-first-migration) cli, +options and commands are just the same. + +#### NOTE run migrations + +This command should be executed after each upgrade to new Horizon version. + +### Run backend + +To start backend server you need to execute following command: + +```console +$ python -m horizon.backend --host 0.0.0.0 --port 8000 +... +``` + +This is a thin wrapper around [uvicorn](https://www.uvicorn.org/#command-line-options) cli, +options and commands are just the same. + +After server is started and ready, open [http://localhost:8000/docs](http://localhost:8000/docs). + +### Add admin users + +To promote specific users to `SUPERADMIN` role, run the following script: + +```console +$ python -m horizon.backend.scripts.manage_admins add admin1 admin2 +... +``` + +See [Scripts][scripts] documentation. diff --git a/mddocs/en/backend/openapi.md b/mddocs/en/backend/openapi.md new file mode 100644 index 00000000..8c6994b9 --- /dev/null +++ b/mddocs/en/backend/openapi.md @@ -0,0 +1,30 @@ +# OpenAPI specification { #backend-openapi } + + + diff --git a/mddocs/en/backend/scripts/index.md b/mddocs/en/backend/scripts/index.md new file mode 100644 index 00000000..c842f892 --- /dev/null +++ b/mddocs/en/backend/scripts/index.md @@ -0,0 +1,5 @@ +# Scripts { #scripts } + +Horizon includes several utility scripts to manage the application at a low level. + +* [Manage Admins][manage-admins-script] diff --git a/mddocs/en/backend/scripts/manage_admins.md b/mddocs/en/backend/scripts/manage_admins.md new file mode 100644 index 00000000..c4ee0300 --- /dev/null +++ b/mddocs/en/backend/scripts/manage_admins.md @@ -0,0 +1,7 @@ +# Manage Admins { #manage-admins-script } + +```argparse + :module: horizon.backend.scripts.manage_admins + :func: create_parser + :prog: python -m horizon.backend.scripts.manage_admins +``` diff --git a/mddocs/en/changelog.md b/mddocs/en/changelog.md new file mode 100644 index 00000000..307e7141 --- /dev/null +++ b/mddocs/en/changelog.md @@ -0,0 +1,19 @@ +# Changelog { #Changelog } + +* [1.1.2 (2025-04-07)][1.1.2] +* [1.1.2 (2025-04-07)][1.1.2] +* [1.1.1 (2025-01-28)][1.1.1] +* [1.0.2 (2024-11-21)][1.0.2] +* [1.0.1 (2024-06-27)][1.0.1] +* [1.0.0 (2024-06-10)][1.0.0] +* [0.2.1 (2024-05-29)][0.2.1] +* [0.2.0 (2024-05-15)][0.2.0] +* [0.1.3 (2024-05-02)][0.1.3] +* [0.1.2 (2024-04-02)][0.1.2] +* [0.1.1 (2024-03-27)][0.1.1] +* [0.0.13 (2024-02-13)][0.0.13] +* [0.0.12 (2024-01-24)][0.0.12] +* [0.0.11 (2024-01-22)][0.0.11] +* [0.0.10 (2024-01-22)][0.0.10] +* [0.0.9 (2024-01-19)][0.0.9] +* [0.0.8 (2024-01-18)][0.0.8] diff --git a/mddocs/en/changelog/0.0.10.md b/mddocs/en/changelog/0.0.10.md new file mode 100644 index 00000000..d2bc8d98 --- /dev/null +++ b/mddocs/en/changelog/0.0.10.md @@ -0,0 +1,5 @@ +# 0.0.10 (2024-01-22) { #0.0.10 } + +## Bug Fixes + +- Update `starlette-exporter` and `uvicorn` to latest versions. diff --git a/mddocs/en/changelog/0.0.11.md b/mddocs/en/changelog/0.0.11.md new file mode 100644 index 00000000..4d84db4a --- /dev/null +++ b/mddocs/en/changelog/0.0.11.md @@ -0,0 +1,5 @@ +# 0.0.11 (2024-01-22) { #0.0.11 } + +## Bug Fixes + +- Remove `starlette` from list of packages installed with `client-sync` extras. diff --git a/mddocs/en/changelog/0.0.12.md b/mddocs/en/changelog/0.0.12.md new file mode 100644 index 00000000..eccb99ea --- /dev/null +++ b/mddocs/en/changelog/0.0.12.md @@ -0,0 +1,5 @@ +# 0.0.12 (2024-01-24) { #0.0.12 } + +## Bug Fixes + +- Fix client compatibility with `urllib` 1.x . diff --git a/mddocs/en/changelog/0.0.13.md b/mddocs/en/changelog/0.0.13.md new file mode 100644 index 00000000..5edcd885 --- /dev/null +++ b/mddocs/en/changelog/0.0.13.md @@ -0,0 +1,5 @@ +# 0.0.13 (2024-02-13) { #0.0.13 } + +## Features + +- `Horizon` is open-source now. diff --git a/mddocs/en/changelog/0.0.8.md b/mddocs/en/changelog/0.0.8.md new file mode 100644 index 00000000..f310586a --- /dev/null +++ b/mddocs/en/changelog/0.0.8.md @@ -0,0 +1,6 @@ +# 0.0.8 (2024-01-18) { #0.0.8 } + +## Features + +- Added retry configuration support to `HorizonClientSync`. ([#DOP-12454](https://github.com/MobileTeleSystems/horizon/issues/DOP-12454)) +- Added `timeout` config parameter support to `HorizonClientSync`. ([#DOP-12545](https://github.com/MobileTeleSystems/horizon/issues/DOP-12545)) diff --git a/mddocs/en/changelog/0.0.9.md b/mddocs/en/changelog/0.0.9.md new file mode 100644 index 00000000..c3fb9c93 --- /dev/null +++ b/mddocs/en/changelog/0.0.9.md @@ -0,0 +1,5 @@ +# 0.0.9 (2024-01-19) { #0.0.9 } + +## Bug Fixes + +- Fix empty `/monitoring/metrics` data. diff --git a/mddocs/en/changelog/0.1.1.md b/mddocs/en/changelog/0.1.1.md new file mode 100644 index 00000000..43857e89 --- /dev/null +++ b/mddocs/en/changelog/0.1.1.md @@ -0,0 +1,27 @@ +# 0.1.1 (2024-03-27) { #0.1.1 } + +## Breaking Changes + +Users now required to explicitly have a role assigned within a namespace to manipulate HWMs as they could before. These changes enforce stricter access control and better management of user permissions within the system. + +- Add role model to Horizon, documentation available at [Role Permissions](../design/permissions.md#role-permissions). ([#27](https://github.com/MobileTeleSystems/horizon/issues/27), [#31](https://github.com/MobileTeleSystems/horizon/issues/31)) +- Restrict deletion of `Namespace` if there are any `hwms` related to it. ([#25](https://github.com/MobileTeleSystems/horizon/issues/25)) + +## Features + +- Add `Namespace History`. Now it is possible to view paginated history of actions for specific namespace. ([#24](https://github.com/MobileTeleSystems/horizon/issues/24)) +- Add `owner_id` field to `Namespace` model to keep track of the owner of the namespace. ([#26](https://github.com/MobileTeleSystems/horizon/issues/26)) +- Add support for managing `SUPERADMIN` roles. ([#36](https://github.com/MobileTeleSystems/horizon/issues/36)) +- Permissions Management: + - Add new API endpoint `PATCH /namespace/:id/permissions` for updating the permissions of users within a namespace. + - Add new API endpoint `GET /namespace/:id/permissions` for fetching the permissions of users within a specific namespace. + - Extend the Python client library with methods `get_namespace_permissions` and `update_namespace_permissions` to interact with the new API endpoints. ([#29](https://github.com/MobileTeleSystems/horizon/issues/29)) +- High Water Marks (HWMs) Management: + - Add new API endpoint `DELETE /hwm/` for bulk deletion of High Water Marks (HWMs) by namespace_id and a list of hwm_ids. + - Extend the Python client library with the method `bulk_delete_hwm` to interact with the new bulk delete HWM API endpoint. ([#37](https://github.com/MobileTeleSystems/horizon/issues/37)) + - Add new API endpoint `POST /hwm/copy` endpoint for copying HWMs between namespaces, with optional history copying. + - Extend the Python client library with the method `copy_hwms` to support the new HWM copy functionality. ([#42](https://github.com/MobileTeleSystems/horizon/issues/42)) + +## Improvements + +- Fix documentation examples. Make documentation more user-friendly. ([#20](https://github.com/MobileTeleSystems/horizon/issues/20)) diff --git a/mddocs/en/changelog/0.1.2.md b/mddocs/en/changelog/0.1.2.md new file mode 100644 index 00000000..2c2e7db2 --- /dev/null +++ b/mddocs/en/changelog/0.1.2.md @@ -0,0 +1,11 @@ +# 0.1.2 (2024-04-02) { #0.1.2 } + +## Features + +- Add new environment variable `HORIZON__ENTRYPOINT__ADMIN_USERS` to Docker image entrypoint. + Here you can pass of usernames which should be automatically promoted to `SUPERADMIN` role during backend startup. ([#45](https://github.com/MobileTeleSystems/horizon/issues/45)) + +## Improvements + +- Improve logging in `manage_admins` script. ([#46](https://github.com/MobileTeleSystems/horizon/issues/46)) +- Fix Pydantic v2 model warnings while starting backend. ([#47](https://github.com/MobileTeleSystems/horizon/issues/47)) diff --git a/mddocs/en/changelog/0.1.3.md b/mddocs/en/changelog/0.1.3.md new file mode 100644 index 00000000..2aa05fe1 --- /dev/null +++ b/mddocs/en/changelog/0.1.3.md @@ -0,0 +1,6 @@ +# 0.1.3 (2024-05-02) { #0.1.3 } + +## Improvements + +- Properly handle `SIGTERM` signals in Docker image entrypoint. +- Update dependencies diff --git a/mddocs/en/changelog/0.2.0.md b/mddocs/en/changelog/0.2.0.md new file mode 100644 index 00000000..aa891201 --- /dev/null +++ b/mddocs/en/changelog/0.2.0.md @@ -0,0 +1,16 @@ +# 0.2.0 (2024-05-15) { #0.2.0 } + +## Breaking Changes + +- Rename `/v1/namespace/:id/permissions` endpoint to `/v1/namespaces/:id/permissions`. ([#61](https://github.com/MobileTeleSystems/horizon/issues/61)) + +## Features + +- Allow using Horizon with multiple uvicorn workers ([#60](https://github.com/MobileTeleSystems/horizon/issues/60)): + - Add `pid` to log formatters + - Add `PROMETHEUS_MULTIPROC_DIR` to `docker-compose.yml` example + +## Bug Fixes + +- Use connection timeout while creating LDAP connections in the pool. ([#58](https://github.com/MobileTeleSystems/horizon/issues/58)) +- Fix response schema for invalid JSON input. diff --git a/mddocs/en/changelog/0.2.1.md b/mddocs/en/changelog/0.2.1.md new file mode 100644 index 00000000..debc8aae --- /dev/null +++ b/mddocs/en/changelog/0.2.1.md @@ -0,0 +1,7 @@ +# 0.2.1 (2024-05-29) { #0.2.1 } + +## Improvements + +- Fix LDAP connection pool configuration example. +- Update uvicorn to 0.30.0, including new multiprocessing workers manager. +- Update dependencies. diff --git a/mddocs/en/changelog/1.0.0.md b/mddocs/en/changelog/1.0.0.md new file mode 100644 index 00000000..115bab51 --- /dev/null +++ b/mddocs/en/changelog/1.0.0.md @@ -0,0 +1,7 @@ +# 1.0.0 (2024-06-10) { #1.0.0 } + +First production-ready release! + +## Improvements + +- Update dependencies diff --git a/mddocs/en/changelog/1.0.1.md b/mddocs/en/changelog/1.0.1.md new file mode 100644 index 00000000..73e6aa51 --- /dev/null +++ b/mddocs/en/changelog/1.0.1.md @@ -0,0 +1,10 @@ +# 1.0.1 (2024-06-27) { #1.0.1 } + +## Dependencies + +- Bump minimal `urllib3` version to `1.26.0`, to avoid exceptions like: + +```default +ValidationError: 1 validation error for HorizonClientSync__root__ + __init__() got an unexpected keyword argument 'allowed_methods' (type=type_error) +``` diff --git a/mddocs/en/changelog/1.0.2.md b/mddocs/en/changelog/1.0.2.md new file mode 100644 index 00000000..71b47d12 --- /dev/null +++ b/mddocs/en/changelog/1.0.2.md @@ -0,0 +1,34 @@ +# 1.0.2 (2024-11-21) { #1.0.2 } + +## Bug fixes + +- Previously client after receiving 4xx responses from the server, raised `requests.exceptions.HTTPError` like: + + ```pycon + >>> client.update_namespace_permissions(namespace_id=234, changes=to_update) + Traceback (most recent call last): + File "horizon/horizon/client/base.py", line 135, in _handle_response + response.raise_for_status() + File "horizon/.venv/lib/python3.12/site-packages/requests/models.py", line 1021, in raise_for_status + raise HTTPError(http_error_msg, response=self) + requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http://localhost:8000/v1/namespaces/234/permissions + ``` + + Now it wraps all these exceptions with `horizon.commons.exceptions` classes, like: + + ```pycon + >>> client.update_namespace_permissions(namespace_id=234, changes=to_update) + Traceback (most recent call last): + File "", line 1, in + File "horizon/horizon/client/sync.py", line 914, in update_namespace_permissions + return self._request( # type: ignore[return-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "horizon/horizon/client/sync.py", line 1031, in _request + return self._handle_response(response, response_class) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "horizon/horizon/client/base.py", line 170, in _handle_response + raise get_exception() from http_exception + horizon.commons.exceptions.entity.EntityNotFoundError: Namespace with id=234 not found + ``` + + `horizon.commons.exceptions` exception types were documented long time ago, so this is not a breaking change, but a bug fix. diff --git a/mddocs/en/changelog/1.1.1.md b/mddocs/en/changelog/1.1.1.md new file mode 100644 index 00000000..643b3c2a --- /dev/null +++ b/mddocs/en/changelog/1.1.1.md @@ -0,0 +1,9 @@ +# 1.1.1 (2025-01-28) { #1.1.1 } + +## Improvements + +- Add compatibility with `Python 3.13` ([#94](https://github.com/MobileTeleSystems/horizon/issues/94)) +- Replace outdated `python-jose` dependency with `authlib.jose`, to fix security issues. ([#97](https://github.com/MobileTeleSystems/horizon/issues/97)) + +Note: preliminary release 1.1.0 was yanked from PyPI because it has wrong `horizon.__version__` value. +The source code of 1.1.1 and dependencies are just the same as 1.1.0. diff --git a/mddocs/en/changelog/1.1.2.md b/mddocs/en/changelog/1.1.2.md new file mode 100644 index 00000000..b1c3cbcb --- /dev/null +++ b/mddocs/en/changelog/1.1.2.md @@ -0,0 +1,7 @@ +# 1.1.2 (2025-04-07) { #1.1.2 } + +## Improvements + +- Reduce image size down to x3. +- Change docker image user from `root` to `horizon`, to improve security. +- SBOM file is generated on release. diff --git a/mddocs/en/changelog/DRAFT.md b/mddocs/en/changelog/DRAFT.md new file mode 100644 index 00000000..4aa1c1bf --- /dev/null +++ b/mddocs/en/changelog/DRAFT.md @@ -0,0 +1,4 @@ + +```{eval-rst} +.. towncrier-draft-entries:: |release| [UNRELEASED] +``` diff --git a/mddocs/en/changelog/NEXT_RELEASE.md b/mddocs/en/changelog/NEXT_RELEASE.md new file mode 100644 index 00000000..a9831f9d --- /dev/null +++ b/mddocs/en/changelog/NEXT_RELEASE.md @@ -0,0 +1 @@ +% towncrier release notes start diff --git a/mddocs/en/changelog/index.md b/mddocs/en/changelog/index.md new file mode 100644 index 00000000..be216b27 --- /dev/null +++ b/mddocs/en/changelog/index.md @@ -0,0 +1,18 @@ +# Changelog + +* [1.1.2 (2025-04-07)][1.1.2] +* [1.1.1 (2025-01-28)][1.1.1] +* [1.0.2 (2024-11-21)][1.0.2] +* [1.0.1 (2024-06-27)][1.0.1] +* [1.0.0 (2024-06-10)][1.0.0] +* [0.2.1 (2024-05-29)][0.2.1] +* [0.2.0 (2024-05-15)][0.2.0] +* [0.1.3 (2024-05-02)][0.1.3] +* [0.1.2 (2024-04-02)][0.1.2] +* [0.1.1 (2024-03-27)][0.1.1] +* [0.0.13 (2024-02-13)][0.0.13] +* [0.0.12 (2024-01-24)][0.0.12] +* [0.0.11 (2024-01-22)][0.0.11] +* [0.0.10 (2024-01-22)][0.0.10] +* [0.0.9 (2024-01-19)][0.0.9] +* [0.0.8 (2024-01-18)][0.0.8] diff --git a/mddocs/en/changelog/next_release/.keep b/mddocs/en/changelog/next_release/.keep new file mode 100644 index 00000000..e69de29b diff --git a/mddocs/en/client/auth.md b/mddocs/en/client/auth.md new file mode 100644 index 00000000..74b2f786 --- /dev/null +++ b/mddocs/en/client/auth.md @@ -0,0 +1,28 @@ +# Auth { #client-auth } + +These classes are used for adding auth information to requests send from client. + + +::: horizon.client.auth.LoginPassword + options: + members: + - login + - password + +::: horizon.client.auth.AccessToken + options: + members: + - token diff --git a/mddocs/en/client/exceptions.md b/mddocs/en/client/exceptions.md new file mode 100644 index 00000000..5516606b --- /dev/null +++ b/mddocs/en/client/exceptions.md @@ -0,0 +1,270 @@ +# Exceptions { #client-exceptions } + +These exception classes are used in client implementations. + +## Base + +### *class* horizon.commons.exceptions.base.ApplicationError + +Base class for all exceptions raised by Horizon. + +* **Attributes:** + [`details`][horizon.commons.exceptions.base.ApplicationError.details] + Details related to specific error + + [`message`][horizon.commons.exceptions.base.ApplicationError.message] + Message string + + + +#### *abstract property* details *: Any* + +Details related to specific error + + + +#### *abstract property* message *: str* + +Message string + + + +## Authorization + +### *class* horizon.commons.exceptions.auth.AuthorizationError(message: str, details: Any = None) + +Authorization request is failed. + +* **Attributes:** + [`details`][horizon.commons.exceptions.auth.AuthorizationError.details] + Details related to specific error + + [`message`][horizon.commons.exceptions.auth.AuthorizationError.message] + Message string + +### Examples + +```pycon +>>> from horizon.commons.exceptions import AuthorizationError +>>> raise AuthorizationError("User 'test' is disabled") +Traceback (most recent call last): +horizon.commons.exceptions.auth.AuthorizationError: User 'test' is disabled +``` + + + +#### *property* details *: Any* + +Details related to specific error + + + +#### *property* message *: str* + +Message string + + + +## Permissions + +### *class* horizon.commons.exceptions.permission.PermissionDeniedError(required_role: str, actual_role: str) + +Permission denied for performing the requested action. + +* **Attributes:** + [`details`][horizon.commons.exceptions.permission.PermissionDeniedError.details] + Details related to specific error + + [`message`][horizon.commons.exceptions.permission.PermissionDeniedError.message] + Message string + +### Examples permissions + +```pycon +>>> from horizon.commons.exceptions import PermissionDeniedError +>>> raise PermissionDeniedError(required_role="DEVELOPER", actual_role="GUEST") +Traceback (most recent call last): +horizon.commons.exceptions.PermissionDeniedError: Permission denied. User has role GUEST but action requires at least DEVELOPER. +``` + + + +#### required_role *: str* + +Required role to perform action + + + +#### actual_role *: str* + +Actual user role + + + +#### *property* message permission *: str* + +Message string + + + +#### *property* details *: dict[str, Any]* + +Details related to specific error + + + +### *class* horizon.commons.exceptions.bad_request.BadRequestError(reason: str) + +Bad request error. + +This exception should be raised when a request cannot be processed due to +client-side errors (e.g., invalid data, duplicate entries). + +### Examples bad request + +```pycon +>>> from horizon.commons.exceptions import BadRequestError +>>> raise BadRequestError("Duplicate username detected. Each username must appear only once.") +Traceback (most recent call last): +horizon.commons.exceptions.BadRequestError: Duplicate username detected. Each username must appear only once. +``` + + + +#### reason *: str* + +Bad request reason message + + +## Entity + +### *class* horizon.commons.exceptions.entity.EntityNotFoundError(entity_type: str, field: str, value: Any) + +Entity not found. + +* **Attributes:** + [`details`][horizon.commons.exceptions.entity.EntityNotFoundError.details] + Details related to specific error + + [`message`][horizon.commons.exceptions.entity.EntityNotFoundError.message] + Message string + +### Examples entity + +```pycon +>>> from horizon.commons.exceptions import EntityNotFoundError +>>> raise EntityNotFoundError("User", "username", "test") +Traceback (most recent call last): +horizon.commons.exceptions.entity.EntityNotFoundError: User with username='test' not found +``` + + + +#### entity_type *: str* + +Entity type + + + +#### field *: str* + +Entity identifier field + + + +#### value *: Any* + +Entity identifier value + + + +#### *property* message entity *: str* + +Message string + + + +#### *property* details entity *: dict[str, Any]* + +Details related to specific error + + + +### *class* horizon.commons.exceptions.entity.EntityAlreadyExistsError(entity_type: str, field: str, value: Any) + +Entity with same identifier already exists. + +* **Attributes:** + [`details`][horizon.commons.exceptions.entity.EntityAlreadyExistsError.details] + Details related to specific error + + [`message`][horizon.commons.exceptions.entity.EntityAlreadyExistsError.message] + Message string + +### Examples entity 2 + +```pycon +>>> from horizon.commons.exceptions import EntityNotFoundError +>>> raise EntityAlreadyExistsError("User", "username", "test") +Traceback (most recent call last): +horizon.commons.exceptions.entity.EntityAlreadyExistsError: User with username='test' already exists +``` + + + +#### entity_type 2 *: str* + +Entity type + + + +#### field 2 *: str* + +Entity identifier field + + + +#### value 2 *: Any* + +Entity identifier value + + + +#### *property* message 2 *: str* + +Message string + + + +#### *property* details 2 *: dict[str, Any]* + +Details related to specific error + + + +## Service + +### *class* horizon.commons.exceptions.service.ServiceError(message: str) + +Service used by application have not responded properly. + +* **Attributes:** + [`message`][horizon.commons.exceptions.service.ServiceError.message] + Message string + +### Examples service + +```pycon +>>> from horizon.commons.exceptions import ServiceError +>>> raise ServiceError("Some server response is invalid") +Traceback (most recent call last): +horizon.commons.exceptions.service.ServiceError: Some server response is invalid +``` + + + +#### *property* message service *: str* + +Message string + + \ No newline at end of file diff --git a/mddocs/en/client/install.md b/mddocs/en/client/install.md new file mode 100644 index 00000000..84fdf1dc --- /dev/null +++ b/mddocs/en/client/install.md @@ -0,0 +1,19 @@ +# Install client { #client-install } + +## Requirements + +- Python 3.7 or above +- Pydantic 1.x or 2.x + +## Installation process + +Install `data-horizon` package with following *extra* dependencies: + +```console +$ pip install data-horizon[client-sync] +... +``` + +Available *extras* are: + +- `client-sync` - [Sync client][client-sync], based on [authlib](https://docs.authlib.org) and [requests](https://requests.readthedocs.io) diff --git a/mddocs/en/client/schemas/hwm.md b/mddocs/en/client/schemas/hwm.md new file mode 100644 index 00000000..515cf593 --- /dev/null +++ b/mddocs/en/client/schemas/hwm.md @@ -0,0 +1,62 @@ +# HWM-related schemas { #client-schemas-hwm } + + + +::: horizon.commons.schemas.v1.hwm + options: + members: + - HWMResponseV1 + - HWMListResponseV1 + - HWMPaginateQueryV1 + - HWMCreateRequestV1 + - HWMUpdateRequestV1 + - HWMBulkCopyRequestV1 + - HWMBulkDeleteRequestV1 + + + + + + + + + + + + + + diff --git a/mddocs/en/client/schemas/hwm_history.md b/mddocs/en/client/schemas/hwm_history.md new file mode 100644 index 00000000..8ffc7725 --- /dev/null +++ b/mddocs/en/client/schemas/hwm_history.md @@ -0,0 +1,26 @@ +# HWM history-related schemas { #client-schemas-hwm-history } + + +::: horizon.commons.schemas.v1.hwm_history + options: + members: + - HWMHistoryResponseV1 + - HWMHistoryPaginateQueryV1 + + + + diff --git a/mddocs/en/client/schemas/index.md b/mddocs/en/client/schemas/index.md new file mode 100644 index 00000000..54c5eb1b --- /dev/null +++ b/mddocs/en/client/schemas/index.md @@ -0,0 +1,16 @@ +# Schemas { #client-schemas-root } + +These classes are used for sending requests to backend and parsing responses. + +All of then are based on Pydantic models. + +## Horizon schemas + +* [Namespace-related schemas][client-schemas-namespace] +* [Namespace history-related schemas][client-schemas-namespace-history] +* [HWM-related schemas][client-schemas-hwm] +* [HWM history-related schemas][client-schemas-hwm-history] +* [Permissions-related schemas][client-schemas-permissions] +* [User-related schemas][client-schemas-user] +* [Ping-related schemas][client-schemas-ping] +* [Pagination-related schemas][client-schemas-pagination] diff --git a/mddocs/en/client/schemas/namespace.md b/mddocs/en/client/schemas/namespace.md new file mode 100644 index 00000000..2a6e9e6e --- /dev/null +++ b/mddocs/en/client/schemas/namespace.md @@ -0,0 +1,39 @@ +# Namespace-related schemas { #client-schemas-namespace } + + + +::: horizon.commons.schemas.v1.namespace + options: + members: + - NamespaceResponseV1 + - NamespacePaginateQueryV1 + - NamespaceCreateRequestV1 + - NamespaceUpdateRequestV1 + + + + + + + + diff --git a/mddocs/en/client/schemas/namespace_history.md b/mddocs/en/client/schemas/namespace_history.md new file mode 100644 index 00000000..da5bcd58 --- /dev/null +++ b/mddocs/en/client/schemas/namespace_history.md @@ -0,0 +1,24 @@ +# Namespace history-related schemas { #client-schemas-namespace-history } + + +::: horizon.commons.schemas.v1.namespace_history + options: + members: + - NamespaceHistoryResponseV1 + - NamespaceHistoryPaginateQueryV1 + + + + diff --git a/mddocs/en/client/schemas/pagination.md b/mddocs/en/client/schemas/pagination.md new file mode 100644 index 00000000..959e5ffe --- /dev/null +++ b/mddocs/en/client/schemas/pagination.md @@ -0,0 +1,32 @@ +# Pagination-related schemas { #client-schemas-pagination } + + +::: horizon.commons.schemas.v1.pagination + options: + members: + - PaginateQueryV1 + - PageResponseV1 + - PageMetaResponseV1 + + + + + + diff --git a/mddocs/en/client/schemas/permissions.md b/mddocs/en/client/schemas/permissions.md new file mode 100644 index 00000000..f5d308ff --- /dev/null +++ b/mddocs/en/client/schemas/permissions.md @@ -0,0 +1,37 @@ +# Permissions-related schemas { #client-schemas-permissions } + + +::: horizon.commons.schemas.v1.permission + options: + members: + - PermissionResponseItemV1 + - PermissionsResponseV1 + - PermissionUpdateRequestItemV1 + - PermissionsUpdateRequestV1 + + + + + + + + diff --git a/mddocs/en/client/schemas/ping.md b/mddocs/en/client/schemas/ping.md new file mode 100644 index 00000000..ef83b92f --- /dev/null +++ b/mddocs/en/client/schemas/ping.md @@ -0,0 +1,16 @@ +# Ping-related schemas { #client-schemas-ping } + + +::: horizon.commons.schemas + options: + members: + - PingResponse + + diff --git a/mddocs/en/client/schemas/user.md b/mddocs/en/client/schemas/user.md new file mode 100644 index 00000000..b12c7fd2 --- /dev/null +++ b/mddocs/en/client/schemas/user.md @@ -0,0 +1,30 @@ +# User-related schemas { #client-schemas-user } + + + + + + + + + + diff --git a/mddocs/en/client/sync.md b/mddocs/en/client/sync.md new file mode 100644 index 00000000..4c6f2a03 --- /dev/null +++ b/mddocs/en/client/sync.md @@ -0,0 +1,129 @@ +# Sync client { #client-sync } + +## Quickstart + +Here is a short example of using sync client to interact with backend. + +Create client object: + +```pycon +>>> from horizon.client.sync import HorizonClientSync +>>> from horizon.client.auth import LoginPassword +>>> client = HorizonClientSync( +... base_url="http://some.domain.com/api", +... auth=LoginPassword(login="me", password="12345"), +... ) +``` + +Check for credentials and issue access token: + +```pycon +>>> client.authorize() +``` + +Create namespace with name "my_namespace": + +```pycon +>>> from horizon.commons.schemas.v1 import NamespaceCreateRequestV1 +>>> created_namespace = client.create_namespace(NamespaceCreateRequestV1(name="my_namespace")) +>>> created_namespace +NamespaceResponseV1( + id=1, + name="my_namespace", + description="", +) +``` + +Create HWM with name "my_hwm" in this namespace: + +```pycon +>>> from horizon.commons.schemas.v1 import HWMCreateRequestV1 +>>> hwm = HWMCreateRequestV1( +... namespace_id=created_namespace.id, +... name="my_hwm", +... type="column_int", +... value=123, +... ) +>>> created_hwm = client.create_hwm(hwm) +>>> created_hwm +HWMResponseV1( + id=1, + namespace_id=1, + name="my_hwm", + description="", + type="column_int", + value=123, + entity="", + expression="", +) +``` + +Update HWM with name "my_hwm" in this namespace: + +```pycon +>>> from horizon.commons.schemas.v1 import HWMUpdateRequestV1 +>>> hwm_change = HWMUpdateRequestV1(value=234) +>>> updated_hwm = client.update_hwm(created_hwm.id, hwm_change) +>>> updated_hwm +HWMResponseV1( + id=1, + namespace_id=1, + name="my_hwm", + description="", + type="column_int", + value=234, + entity="", + expression="", +) +``` + +## Reference + + +::: horizon.client.sync.HorizonClientSync + options: + members: + - authorize + - ping + - whoami + - paginate_namespaces + - get_namespace + - create_namespace + - update_namespace + - delete_namespace + - paginate_hwm + - get_hwm + - create_hwm + - update_hwm + - delete_hwm + - bulk_delete_hwm + - get_namespace_permissions + - update_namespace_permissions + - paginate_hwm_history + - paginate_namespace_history + - retry + - bulk_copy_hwm + +::: horizon.client.sync.RetryConfig + +::: horizon.client.sync.TimeoutConfig diff --git a/mddocs/en/conf.py b/mddocs/en/conf.py new file mode 100644 index 00000000..9cfad4a1 --- /dev/null +++ b/mddocs/en/conf.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: 2025-present MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + + +import os +import sys +from pathlib import Path + +from packaging import version as Version + +PROJECT_ROOT_DIR = Path(__file__).parent.parent.resolve() + +sys.path.insert(0, os.fspath(PROJECT_ROOT_DIR)) + +# -- Project information ----------------------------------------------------- + +project = "horizon" +copyright = "2023-2025 MTS PJSC" +author = "DataOps.ETL" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. + +# this value is updated automatically by `poetry version ...` and poetry-bumpversion plugin +ver = Version.parse("1.1.3") +version = ver.base_version +# The full version, including alpha/beta/rc tags. +release = ver.public + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "numpydoc", + "sphinx_copybutton", + "sphinx.ext.doctest", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + # "sphinxcontrib.autodoc_pydantic", + "sphinxcontrib.towncrier", # provides `towncrier-draft-entries` directive + "sphinx_issues", + "sphinx_design", # provides `dropdown` directive + "sphinxcontrib.plantuml", + "sphinx_favicon", + "sphinxarg.ext", + "sphinx_last_updated_by_git", +] + +swagger = [ + { + "name": "Horizon REST API", + "page": "openapi", + "id": "horizon-api", + "options": { + "url": "_static/openapi.json", + }, + }, +] + +numpydoc_show_class_members = True +autodoc_pydantic_model_show_config = False +autodoc_pydantic_model_show_config_summary = False +autodoc_pydantic_model_show_config_member = False +autodoc_pydantic_model_show_json = False +autodoc_pydantic_model_show_validator_summary = False +autodoc_pydantic_model_show_validator_members = False +autodoc_pydantic_model_member_order = "bysource" +autodoc_pydantic_settings_show_config = False +autodoc_pydantic_settings_show_config_summary = True +autodoc_pydantic_settings_show_config_member = False +autodoc_pydantic_settings_show_json = False +autodoc_pydantic_settings_show_validator_summary = False +autodoc_pydantic_settings_show_validator_members = False +autodoc_pydantic_settings_member_order = "bysource" +autodoc_pydantic_field_list_validators = False +sphinx_tabs_disable_tab_closing = True + +# prevent >>>, ... and doctest outputs from copying +copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +copybutton_prompt_is_regexp = True +copybutton_copy_empty_lines = False +copybutton_only_copy_prompt_lines = True + +towncrier_draft_autoversion_mode = "draft" +towncrier_draft_include_empty = False +towncrier_draft_working_directory = PROJECT_ROOT_DIR + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. + +html_theme = "furo" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "sidebar_hide_name": True, +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] +html_extra_path = ["robots.txt"] +html_css_files = [ + "custom.css", +] + +html_logo = "./_static/logo_no_title.svg" +favicons = [ + {"rel": "icon", "href": "icon.svg", "type": "image/svg+xml"}, +] + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "horizon-doc" + + +# which is the equivalent to: +issues_uri = "https://github.com/MobileTeleSystems/horizon/issues/{issue}" +issues_pr_uri = "https://github.com/MobileTeleSystems/horizon/pulls/{pr}" +issues_commit_uri = "https://github.com/MobileTeleSystems/horizon/commit/{commit}" +issues_user_uri = "https://github.com/{user}" diff --git a/mddocs/en/contributing.md b/mddocs/en/contributing.md new file mode 100644 index 00000000..177f6924 --- /dev/null +++ b/mddocs/en/contributing.md @@ -0,0 +1,383 @@ +# Contributing Guide { #contributing } + +Welcome! There are many ways to contribute, including submitting bug +reports, improving documentation, submitting feature requests, reviewing +new submissions, or contributing code that can be incorporated into the +project. + +## Limitations + +We should keep close to these items during development: + +* Some companies still use Python 3.7. So it is required to keep compatibility if possible, at least for *client* part of package. +* Different users uses Horizon in different ways - someone store data in Postgres, someone in MySQL, some users need LDAP. Such dependencies should be optional. + +## Initial setup for local development + +### Install Git + +Please follow [instruction](https://docs.github.com/en/get-started/quickstart/set-up-git). + +### Create a fork + +If you are not a member of a development team building horizon, you should create a fork before making any changes. + +Please follow [instruction](https://docs.github.com/en/get-started/quickstart/fork-a-repo). + +### Clone the repo + +Open terminal and run these commands: + +```bash +git clone https://github.com/MobileTeleSystems/horizon -b develop + +cd horizon +``` + +### Setup environment + +Firstly, install [make](https://www.gnu.org/software/make/manual/make.html). It is used for running complex commands in local environment. + +Secondly, create virtualenv and install dependencies: + +```bash +make venv-init +``` + +If you already have venv, but need to install dependencies required for development: + +```bash +make venv-install +``` + +We are using [poetry](https://python-poetry.org/docs/managing-dependencies/) for managing dependencies and building the package. +It allows to keep development environment the same for all developers due to using lock file with fixed dependency versions. + +There are *extra* dependencies (included into package as optional): + +* `backend` +* `client-sync` +* `postgres` +* `ldap` + +And *groups* (not included into package, used locally and in CI): + +* `test` - for running tests +* `dev` - for development, like linters, formatters, mypy, pre-commit and so on +* `docs` - for building documentation + +### Enable pre-commit hooks + +[pre-commit](https://pre-commit.com/) hooks allows to validate & fix repository content before making new commit. +It allows to run linters, formatters, fix file permissions and so on. If something is wrong, changes cannot be committed. + +Firstly, install pre-commit hooks: + +```bash +pre-commit install --install-hooks +``` + +Ant then test hooks run: + +```bash +pre-commit run +``` + +## How to + +### Run development instance locally + +Start DB container: + +```bash +make db +``` + +Then start development server: + +```bash +make dev +``` + +And open [http://localhost:8000/docs](http://localhost:8000/docs) + +Settings are stored in `.env.local` file. + +### Working with migrations + +Start database: + +```bash +make db-start +``` + +Generate revision: + +```bash +make db-revision +``` + +Upgrade db to `head` migration: + +```bash +make db-upgrade +``` + +Downgrade db to `head-1` migration: + +```bash +make db-downgrade +``` + +### Run tests locally + +Start all containers with dependencies: + +```bash +make db # for backend & client tests +make ldap-start # for backend tests +make dev # for client test, run in separate terminal tab +``` + +Run tests: + +```bash +make test +``` + +You can pass additional arguments, they will be passed to pytest: + +```bash +make test PYTEST_ARGS="-m client-sync -lsx -vvvv --log-cli-level=INFO" +``` + +Stop all containers and remove created volumes: + +```bash +make cleanup ARGS="-v" +``` + +Get fixtures not used by any test: + +```bash +make check-fixtures +``` + +### Build CI image locally + +This image is build in CI for testing purposes, but you can do that locally as well: + +```bash +make test-build +``` + +### Run production instance locally + +Firstly, build production image: + +```bash +make prod-build +``` + +And then start it: + +```bash +make prod +``` + +Then open [http://localhost:8000/docs](http://localhost:8000/docs) + +Settings are stored in `.env.docker` file. + +### Build documentation + +Build documentation using Sphinx & open it: + +```bash +make docs +``` + +If documentation should be build cleanly instead of reusing existing build result: + +```bash +make docs-fresh +``` + +## Review process + +Please create a new GitHub issue for any significant changes and +enhancements that you wish to make. Provide the feature you would like +to see, why you need it, and how it will work. Discuss your ideas +transparently and get community feedback before proceeding. + +Significant Changes that you wish to contribute to the project should be +discussed first in a GitHub issue that clearly outlines the changes and +benefits of the feature. + +Small Changes can directly be crafted and submitted to the GitHub +Repository as a Pull Request. + +### Create pull request + +Commit your changes: + +```bash +git commit -m "Commit message" +git push +``` + +Then open Github interface and [create pull request](https://docs.github.com/en/get-started/quickstart/contributing-to-projects#making-a-pull-request). +Please follow guide from PR body template. + +After pull request is created, it get a corresponding number, e.g. 123 (`pr_number`). + +### Write release notes + +`horizon` uses [towncrier](https://pypi.org/project/towncrier/) +for changelog management. + +To submit a change note about your PR, add a text file into the +[docs/changelog/next_release](./next_release) folder. It should contain an +explanation of what applying this PR will change in the way +end-users interact with the project. One sentence is usually +enough but feel free to add as many details as you feel necessary +for the users to understand what it means. + +**Use the past tense** for the text in your fragment because, +combined with others, it will be a part of the “news digest” +telling the readers **what changed** in a specific version of +the library *since the previous version*. + +reStructuredText syntax for highlighting code (inline or block), +linking parts of the docs or external sites. +If you wish to sign your change, feel free to add `-- by +:user:`github-username`` at the end (replace `github-username` +with your own!). + +Finally, name your file following the convention that Towncrier +understands: it should start with the number of an issue or a +PR followed by a dot, then add a patch type, like `feature`, +`doc`, `misc` etc., and add `.rst` as a suffix. If you +need to add more than one fragment, you may add an optional +sequence number (delimited with another period) between the type +and the suffix. + +In general the name will follow `..rst` pattern, +where the categories are: + +* `feature`: Any new feature +* `bugfix`: A bug fix +* `improvement`: An improvement +* `doc`: A change to the documentation +* `dependency`: Dependency-related changes +* `misc`: Changes internal to the repo like CI, test and build changes + +A pull request may have more than one of these components, for example +a code change may introduce a new feature that deprecates an old +feature, in which case two fragments should be added. It is not +necessary to make a separate documentation fragment for documentation +changes accompanying the relevant code changes. + +#### Examples for adding changelog entries to your Pull Requests + +```rst +Added a ``:github:user:`` role to Sphinx config -- by :github:user:`someuser` +``` + +```rst +Fixed behavior of ``backend`` -- by :github:user:`someuser` +``` + +```rst +Added support of ``timeout`` in ``LDAP`` +-- by :github:user:`someuser`, :github:user:`anotheruser` and :github:user:`otheruser` +``` + +#### How to skip change notes check? + +Just add `ci:skip-changelog` label to pull request. + +#### Release Process + +Before making a release from the `develop` branch, follow these steps: + +1. Checkout to `develop` branch and update it to the actual state + +```bash +git checkout develop +git pull -p +``` + +1. Backup `NEXT_RELEASE.rst` + +```bash +cp "docs/changelog/NEXT_RELEASE.rst" "docs/changelog/temp_NEXT_RELEASE.rst" +``` + +1. Build the Release notes with Towncrier + +```bash +VERSION=$(poetry version -s) +towncrier build "--version=${VERSION}" --yes +``` + +1. Change file with changelog to release version number + +```bash +mv docs/changelog/NEXT_RELEASE.rst "docs/changelog/${VERSION}.rst" +``` + +1. Remove content above the version number heading in the `${VERSION}.rst` file + +```bash +awk '!/^.*towncrier release notes start/' "docs/changelog/${VERSION}.rst" > temp && mv temp "docs/changelog/${VERSION}.rst" +``` + +1. Update Changelog Index + +```bash +awk -v version=${VERSION} '/DRAFT/{print;print " " version;next}1' docs/changelog/index.rst > temp && mv temp docs/changelog/index.rst +``` + +1. Restore `NEXT_RELEASE.rst` file from backup + +```bash +mv "docs/changelog/temp_NEXT_RELEASE.rst" "docs/changelog/NEXT_RELEASE.rst" +``` + +1. Commit and push changes to `develop` branch + +```bash +git add . +git commit -m "Prepare for release ${VERSION}" +git push +``` + +1. Merge `develop` branch to `master`, **WITHOUT** squashing + +```bash +git checkout master +git pull +git merge develop +git push +``` + +1. Add git tag to the latest commit in `master` branch + +```bash +git tag "$VERSION" +git push origin "$VERSION" +``` + +1. Update version in `develop` branch **after release**: + +```bash +git checkout develop + +NEXT_VERSION=$(echo "$VERSION" | awk -F. '/[0-9]+\./{$NF++;print}' OFS=.) +poetry version "$NEXT_VERSION" + +git add . +git commit -m "Bump version" +git push +``` diff --git a/mddocs/en/design/entities.md b/mddocs/en/design/entities.md new file mode 100644 index 00000000..1f98d274 --- /dev/null +++ b/mddocs/en/design/entities.md @@ -0,0 +1,236 @@ +# Entities { #entities } + +## HWM + +### Description + +High Water Mark (or *HWM* for short) is a record which allows tracking state of operations (usually read). For example: + +- Save max value of a column read from a table (e.g. `updated_at`), and then use it exclude already read rows on the next read. +- Save list of files handled by a process, and then use it to exclude these files on the next read. +- Same max modification time of files handled by a process, and then use it to exclude these files on the next read. + +Each HWM record is bound to a specific Namespace. HWM `name` is unique within this Namespace. + +Users may create unlimited number of HWMs within a namespace. + +### Fields + +HWM record has following fields: + +- `id: integer` - internal HWM identifier, generated using sequence, read only. +- `namespace_id: integer` - bound to a Namespace, mandatory. +- `name: string` - unique name within a Namespace, mandatory. +- `value: json` - value associated with HWM, mandatory. +- `type: string` - any string describing type of `value` field content, mandatory. +- `description: string` - human readable description. +- `entity: string | null` - name of entity (e.g. table name, folder) which value is bound to. +- `expression: string | null` - expression (e.g. column name) used to get value from `entity`. +- `changed_at: datetime` - filled up automatically on each item change, read only. +- `changed_by: User | null` - filled up automatically on each item change, read only. + +`type` and `value` can contain any value, and are not parsed or checked by backend. This allows users to create HWMs of any type in any time, +without patching backend. But it is up to user to keep consistency of these fields. + +### Limitations + +HWM cannot be moved between namespaces. Users have to explicitly create a copy of existing HWM in new namespace, +and delete old HWM. + +## Namespace + +### Description namespace + +Namespace is a container for HWM records. + +Each namespace has an owner which can alter namespace properties, see [Role Permissions][role-permissions]. + +Users may create unlimited number of namespaces. If user created a namespace, it is automatically set as the namespace owner. + +### Fields namespace + +Namespace record has following fields: + +- `id: integer` - internal namespace identifier, generated using sequence, read only. +- `name: string` - unique per Horizon instance, mandatory +- `description: string` - human readable description. +- `owned_by: User` - is set automatically while namespace is created, mandatory. +- `changed_at: datetime` - filled up automatically on each item change, read only. +- `changed_by: User | null` - filled up automatically on each item change, read only. + +## User + +### Description user + +Users are used for: + +- Authentication +- RBAC permissions model +- Keeping track of changes made on Namespace or HWM. + +User records are created automatically after successful authentication. + +### Fields user + +User record has following fields: + +- `id: integer` - internal user identifier, generated using sequence, read only +- `username: string` - unique per Horizon instance +- `is_active: boolean` - flag if user is allowed to log in. +- `is_admin: boolean` - flag for SUPERUSER role. + +### Limitations user + +For now it is not possible to remove user after creation. + +## Permission + +### Description permission + +User can be assigned a Role within a Namespace, which can allow or disallow performing specific operation within this Namespace. +User can have different roles in different namespaces. See [Role Permissions][role-permissions]. + +### Fields permission + +Permission record has following fields: + +- `user: User` +- `namespace: Namespace` +- `role: enum`. + +### Limitations permission + +User can be assigned only one Role within a Namespace. + +## NamespaceHistory + +### Description namespacehistory + +Change of each Namespace value produces a HWMHistory item, which can be used for audit purpose. +History is append-only, items cannot be changed or deleted using API. + +### Fields namespacehistory + +NamespaceHistory record has following fields (all read-only): + +- `id: integer` - internal history item identifier, generated using sequence. +- `namespace_id: integer` - bound to Namespace item. +- `name: string`. +- `description: string`. +- `owned_by: User`. +- `changed_at: datetime` - filled up automatically on each item change. +- `changed_by: User | null` - filled up automatically on each item change. +- `action: string` - change description, e.g. `Created`, `Updated`, `Deleted`. + +## HWMHistory + +### Description HWMhistory + +Change of each HWM value produces a HWMHistory item, which can be used for audit purpose. +History is append-only, items cannot be changed or deleted using API. + +### Fields HWMhistory + +HWMHistory record has following fields (all read-only): + +- `id: integer` - internal history item identifier, generated using sequence. +- `hwm_id: integer` - bound to HWM item. +- `name: string`. +- `value: any | null`. +- `type: string`. +- `description: string`. +- `entity: string | null`. +- `expression: string | null`. +- `changed_at: datetime` - filled up automatically on each item change. +- `changed_by: User | null` - filled up automatically on each item change. +- `action: string` - change description, e.g. `Created`, `Updated`, `Deleted`. + +## Entity Diagram + +```plantuml + + @startuml + title Entity Diagram + + left to right direction + + entity User { + * id + ---- + * username + is_active + is_admin + } + + entity Namespace { + * id + ---- + * namespace_id + * name + * owned_by + description + changed_at + changed_by + } + + entity HWM { + * id + ---- + * name + * type + * value + description + entity + expression + changed_at + changed_by + } + + entity NamespaceHistory { + * id + ---- + * namespace_id + name + owned_by + description + changed_at + changed_by + action + } + + entity HWMHistory { + * id + ---- + * hwm_id + * namespace_id + name + type + value + description + entity + expression + changed_at + changed_by + action + } + + entity Permission { + * user_id + * namespace_id + ---- + * role + } + + HWM ||--o{ Namespace + Namespace }o--o| NamespaceHistory + HWM }o--o| HWMHistory + Namespace "owner" ||--o{ User + Namespace }o--|| Permission + Permission ||--o{ User + + @enduml +``` + +```mermaid + +``` diff --git a/mddocs/en/design/permissions.md b/mddocs/en/design/permissions.md new file mode 100644 index 00000000..fc3427a6 --- /dev/null +++ b/mddocs/en/design/permissions.md @@ -0,0 +1,40 @@ +# Role Permissions { #role-permissions } + +Horizon implements a role-based access control model to manage permissions across different entities in the service. + +## Role Model Overview + +Roles are defined within the context of namespaces, with the exception of the superadmin role. A user can be associated with one, several, or no namespaces at all. + +- **GUEST**: User without a specific namespace assignment, having limited access rights. This is a default role. +- **DEVELOPER**: Users with development-related permissions. +- **MAINTAINER**: Users with permissions similar to developers but with additional rights in certain areas. +- **OWNER**: Users with full permissions within their owned namespaces and associated HWMs. +- **SUPERADMIN**: Users with full system-wide permissions. + +## Namespace Permissions + +| Role | Create | Read | Update | Delete | Manage Users | +|------------|----------|--------|----------|----------|----------------| +| GUEST | `+` | `+` | `-` | `-` | `-` | +| DEVELOPER | `+` | `+` | `-` | `-` | `-` | +| MAINTAINER | `+` | `+` | `-` | `-` | `-` | +| OWNER | `+` | `+` | `+` | `+` | `+` | +| SUPERADMIN | `+` | `+` | `+` | `+` | `+` | + +## HWM Permissions + +| Role | Create | Read | Update | Delete | +|------------|----------|--------|----------|----------| +| GUEST | `-` | `+` | `-` | `-` | +| DEVELOPER | `+` | `+` | `+` | `-` | +| MAINTAINER | `+` | `+` | `+` | `+` | +| OWNER | `+` | `+` | `+` | `+` | +| SUPERADMIN | `+` | `+` | `+` | `+` | + +## Superadmin Role + +The `SUPERADMIN` role grants a user unrestricted access across all entities and operations within the Horizon service. +Users with the `SUPERADMIN` role can create, read, update, delete, and manage users across all `namespaces` and `HWMs` without any restrictions. + +For details on how to update `SUPERADMIN` roles via the command-line script, see the [Manage Admins][manage-admins-script]. diff --git a/mddocs/en/index.md b/mddocs/en/index.md new file mode 100644 index 00000000..4b38c4f9 --- /dev/null +++ b/mddocs/en/index.md @@ -0,0 +1,55 @@ +# Data.Horizon { #readme } + +[![Repo status - Active](https://www.repostatus.org/badges/latest/active.svg)](https://github.com/MobileTeleSystems/horizon) [![DockerHub - Latest release](https://img.shields.io/docker/v/mtsrus/horizon-backend?sort=semver&label=docker)](https://hub.docker.com/r/mtsrus/horizon-backend) [![PyPI - Latest Release](https://img.shields.io/pypi/v/data-horizon)](https://pypi.org/project/data-horizon/) [![PyPI - License](https://img.shields.io/pypi/l/data-horizon.svg)](https://github.com/MobileTeleSystems/horizon/blob/develop/LICENSE.txt) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/data-horizon.svg)](https://pypi.org/project/data-horizon/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/data-horizon)](https://pypi.org/project/data-horizon/) +[![Documentation - ReadTheDocs](https://readthedocs.org/projects/data-horizon/badge/?version=stable)](https://data-horizon.readthedocs.io/) [![Github Actions - latest CI build status](https://github.com/MobileTeleSystems/horizon/workflows/Tests/badge.svg)](https://github.com/MobileTeleSystems/horizon/actions) [![Test coverage - percent](https://codecov.io/gh/MobileTeleSystems/horizon/branch/develop/graph/badge.svg?token=BIRWPTWEE0)](https://codecov.io/gh/MobileTeleSystems/horizon) [![pre-commit.ci - status](https://results.pre-commit.ci/badge/github/MobileTeleSystems/horizon/develop.svg)](https://results.pre-commit.ci/latest/github/MobileTeleSystems/horizon/develop) + +![Horizon logo](_static/logo.svg) + +## What is Data.Horizon? + +Data.Horizon is an application that implements simple HWM Store. Right now it includes: + +* REST API +* Python client + +## Goals + +* Allow users to save and fetch High Water Mark (*HWM*) items. These are `name+type+value` triples with few optional fields. +* Avoid confusion between different user’s data by separating HWMs to different *namespaces*. Each HWM is bound to namespace. +* Allow users to get HWM change history, to determine who and when changed a specific HWM value and other fields. +* Provide RBAC model to ensure that interaction with `HWMs` and `Namespaces` are governed by role assigned to each user. Roles are assigned per namespace. + +## Non-goals + +* This is not a *data* storage, it is not designed to store raw table rows. It is designed to store only HWM values. +* Attaching machine-readable metadata for HWMs (like `process`, `origin`) is not supported. This should be stored somewhere else. + +## Horizon + +High-level design + +* [entities][entities] +* [permissions][permissions] + +Backend + +* [install][backend-install] +* [architecture][backend-architecture] +* [configuration][backend-configuration] +* [auth][backend-auth-providers] +* [openapi][backend-openapi] +* [scripts][scripts] + +Client + +* [install][client-install] +* [sync][client-sync] +* [auth][client-auth] +* [schemas][client-schemas-root] +* [exceptions][client-exceptions] + +Development + +* [changelog][changelog] +* [contributing][contributing] +* [security][security] diff --git a/mddocs/en/make.bat b/mddocs/en/make.bat new file mode 100644 index 00000000..53ad1e82 --- /dev/null +++ b/mddocs/en/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/mddocs/en/robots.txt b/mddocs/en/robots.txt new file mode 100644 index 00000000..46f730d1 --- /dev/null +++ b/mddocs/en/robots.txt @@ -0,0 +1,5 @@ +User-agent: * +Allow: /*/stable/ +Allow: /en/stable/ # Fallback for bots that don't understand wildcards +Disallow: / +Sitemap: https://data-horizon.readthedocs.io/sitemap.xml \ No newline at end of file diff --git a/mddocs/en/security.md b/mddocs/en/security.md new file mode 100644 index 00000000..b5880bb8 --- /dev/null +++ b/mddocs/en/security.md @@ -0,0 +1,26 @@ +# Security { #security } + +## Supported Python versions + +* Client: 3.7 or above +* Backend: 3.11 or above + +## Product development security recommendations + +1. Update dependencies to last stable version +2. Build SBOM for the project +3. Perform SAST (Static Application Security Testing) where possible + +## Product development security requirements + +1. No binaries in repository +2. No passwords, keys, access tokens in source code +3. No “Critical” and/or “High” vulnerabilities in contributed source code + +## Vulnerability reports + +Please, use email [mailto:onetools@mts.ru](mailto:onetools@mts.ru) for reporting security issues or anything that can cause any consequences for security. + +Please avoid any public disclosure (including registering issues) at least until it is fixed. + +Thank you in advance for understanding.