From f74a4779aad81658eeda311cf184bdbc92ea905c Mon Sep 17 00:00:00 2001 From: optclblast Date: Sat, 14 Dec 2024 02:42:59 -0500 Subject: [PATCH] tmp --- Makefile | 2 - README.md | 6 ++- Taskfile.yaml | 10 +++++ compose.rw.yaml | 41 +++++++++++++++++++ docker-compose.yaml | 15 ------- internal/app/upload_file.go | 6 +-- internal/files_engine/engine.go | 16 ++++++-- internal/files_engine/files.go | 7 ---- internal/storage/interface.go | 3 +- internal/storage/models/files/files.go | 11 +++++ internal/storage/postgres/connection_pool.go | 35 ++++++++++++++++ .../20241214071428_init.sql | 13 +++--- 12 files changed, 128 insertions(+), 37 deletions(-) delete mode 100644 Makefile create mode 100644 Taskfile.yaml create mode 100644 compose.rw.yaml delete mode 100644 docker-compose.yaml delete mode 100644 internal/files_engine/files.go create mode 100644 internal/storage/models/files/files.go create mode 100644 internal/storage/postgres/connection_pool.go rename schema.sql => migrations/20241214071428_init.sql (92%) diff --git a/Makefile b/Makefile deleted file mode 100644 index 6f6e29f..0000000 --- a/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -run: - go run cmd/main.go \ No newline at end of file diff --git a/README.md b/README.md index 10f4579..1dd0304 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ # DrainCloud Core DrainCloud Core is an all-in-one lightweight DrainCloud distribution designed to work in resource-constrained environments. -It requires **# TODO put requirements here ** +The node can work in three modes: #TBD +1. All-in-one mode, the recommended one. +2. Auth-node. Only auth api will be operational. +3. Storage-node. Only filestorage api will be operational. + diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 0000000..df2ec47 --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,10 @@ +version: 3 + +tasks: + prepare-env: + cmds: + - sudo docker swarm init + - sudo docker stack deploy draincloud_core -c ./compose.rw.yaml + migrate-local-status: + cmds: + - goose postgres "postgres://draincloud:draincloud@localhost:5432/draincloud" status -dir migrations \ No newline at end of file diff --git a/compose.rw.yaml b/compose.rw.yaml new file mode 100644 index 0000000..5571d68 --- /dev/null +++ b/compose.rw.yaml @@ -0,0 +1,41 @@ +services: + rw_1: + image: postgres:17 + container_name: draincloud-db-rw-1 + ports: + - 5432:5432 + environment: + - POSTGRES_USER=draincloud + - POSTGRES_PASSWORD=draincloud + - POSTGRES_DB=draincloud + volumes: + - draincloud-rw-1:/var/lib/postgresql/data + + rw_2: + image: postgres:17 + container_name: draincloud-db-rw-2 + ports: + - 5433:5432 + environment: + - POSTGRES_USER=draincloud + - POSTGRES_PASSWORD=draincloud + - POSTGRES_DB=draincloud + volumes: + - draincloud-rw-2:/var/lib/postgresql/data + + rw_3: + image: postgres:17 + container_name: draincloud-db-rw-3 + ports: + - 5434:5432 + environment: + - POSTGRES_USER=draincloud + - POSTGRES_PASSWORD=draincloud + - POSTGRES_DB=draincloud + volumes: + - draincloud-rw-3:/var/lib/postgresql/data + +volumes: + draincloud-rw-1: {} + draincloud-rw-2: {} + draincloud-rw-3: {} \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 0425d10..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,15 +0,0 @@ -services: - database: - image: postgres:17 - container_name: draincloud-db - ports: - - 5432:5432 - environment: - - POSTGRES_USER=draincloud - - POSTGRES_PASSWORD=draincloud - - POSTGRES_DB=draincloud - volumes: - - draincloud-db-data:/var/lib/postgresql/data - -volumes: - draincloud-db-data: {} \ No newline at end of file diff --git a/internal/app/upload_file.go b/internal/app/upload_file.go index 9acd81b..0863114 100644 --- a/internal/app/upload_file.go +++ b/internal/app/upload_file.go @@ -70,11 +70,11 @@ func (d *DrainCloud) uploadFile(ctx *gin.Context, userID int64) error { return nil } -func parseExtension(filename string) filesengine.FileExtension { +func parseExtension(filename string) string { parts := strings.Split(filename, ".") if len(parts) == 0 { - return filesengine.FileExtensionUnspecified + return "" } - return filesengine.FileExtension(parts[len(parts)-1]) + return parts[len(parts)-1] } diff --git a/internal/files_engine/engine.go b/internal/files_engine/engine.go index f96b245..0866b0d 100644 --- a/internal/files_engine/engine.go +++ b/internal/files_engine/engine.go @@ -2,8 +2,10 @@ package filesengine import ( "context" + "fmt" "git.optclblast.xyz/draincloud/draincloud-core/internal/storage" + "git.optclblast.xyz/draincloud/draincloud-core/internal/storage/models/files" ) type FilesEngine struct { @@ -24,17 +26,25 @@ func NewFilesEngine( type File struct { Name string UserID int64 - Ext FileExtension + Ext string Type string Size int64 Data []byte } +// TODO save file func (e *FilesEngine) SaveFile( ctx context.Context, file File, ) (int64, error) { - e.metaStorage.SaveMetadata(ctx, file.Type, ) + fileID, err := e.metaStorage.SaveMetadata(ctx, files.FileMetadata{}) + if err != nil { + return -1, fmt.Errorf("failed to create new file metadata: %w", err) + } - return -1, nil + if err = e.blobStorage.SaveBlob(ctx, fileID, file.Data); err != nil { + return -1, fmt.Errorf("failed to save file data: %w", err) + } + + return fileID, nil } diff --git a/internal/files_engine/files.go b/internal/files_engine/files.go deleted file mode 100644 index f9a45d4..0000000 --- a/internal/files_engine/files.go +++ /dev/null @@ -1,7 +0,0 @@ -package filesengine - -type FileExtension string - -const ( - FileExtensionUnspecified FileExtension = "unspecified" -) diff --git a/internal/storage/interface.go b/internal/storage/interface.go index 3392dc0..2fc922f 100644 --- a/internal/storage/interface.go +++ b/internal/storage/interface.go @@ -6,6 +6,7 @@ import ( "git.optclblast.xyz/draincloud/draincloud-core/internal/storage/models" auditmodels "git.optclblast.xyz/draincloud/draincloud-core/internal/storage/models/audit" + "git.optclblast.xyz/draincloud/draincloud-core/internal/storage/models/files" ) type Database interface { @@ -27,7 +28,7 @@ type AuthAuditLogStorage interface { } type MetaStorage interface { - SaveMetadata(ctx context.Context, fileType string, size int64, ext string) (int64, error) + SaveMetadata(ctx context.Context, meta files.FileMetadata) (int64, error) } type BlobStorage interface { diff --git a/internal/storage/models/files/files.go b/internal/storage/models/files/files.go new file mode 100644 index 0000000..599b07e --- /dev/null +++ b/internal/storage/models/files/files.go @@ -0,0 +1,11 @@ +package files + +type FileMetadata struct { + Id int64 + Name string + UserID int64 + Ext string + Type string + FSLink string + Size int64 +} diff --git a/internal/storage/postgres/connection_pool.go b/internal/storage/postgres/connection_pool.go new file mode 100644 index 0000000..b3f037f --- /dev/null +++ b/internal/storage/postgres/connection_pool.go @@ -0,0 +1,35 @@ +package postgres + +import ( + "context" + "log/slog" + "sync" + + "git.optclblast.xyz/draincloud/draincloud-core/internal/logger" + "github.com/jackc/pgx/v5" +) + +type ShardMap = map[uint16]*pgx.ConnConfig + +type ShardCluster struct { + m sync.Mutex + shards map[uint16]*pgx.Conn +} + +func NewShardCluster(ctx context.Context, shardMap ShardMap) *ShardCluster { + shards := make(map[uint16]*pgx.Conn, len(shardMap)) + for n, cfg := range shardMap { + conn, err := pgx.ConnectConfig(ctx, cfg) + if err != nil { + logger.Fatal(ctx, "failed to connect to shard", slog.Uint64("num", uint64(n)), logger.Err(err)) + } + shards[n] = conn + } + return &ShardCluster{shards: shards} +} + +func (c *ShardCluster) SelectShard(n uint16) *pgx.Conn { + c.m.Lock() + defer c.m.Unlock() + return c.shards[n] +} diff --git a/schema.sql b/migrations/20241214071428_init.sql similarity index 92% rename from schema.sql rename to migrations/20241214071428_init.sql index 2b91e67..e8be9e3 100644 --- a/schema.sql +++ b/migrations/20241214071428_init.sql @@ -1,3 +1,7 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'up SQL query'; + -- Users as auth data create table if not exists users ( id bigserial primary key, @@ -64,9 +68,8 @@ create table directory_users_access ( create index idx_dua_owner_id_parent_dir on directories(owner_id, parent_dir); - - - - - +-- +goose Down +-- +goose StatementBegin +SELECT 'down SQL query'; +-- +goose StatementEnd