From e44448e2c7bad1e0259e161449a6612c5c4ce6eb Mon Sep 17 00:00:00 2001 From: optclblast Date: Mon, 13 May 2024 00:35:29 +0300 Subject: [PATCH] redis cache init --- backend/.vscode/launch.json | 4 +- backend/Makefile | 8 ++- backend/cmd/main.go | 14 ++++ backend/docker-compose.yaml | 10 +++ backend/go.mod | 2 + backend/go.sum | 4 ++ backend/internal/factory/interactors.go | 4 +- backend/internal/factory/repositories.go | 21 ++++++ backend/internal/factory/wire.go | 2 + backend/internal/factory/wire_gen.go | 5 +- .../rest/presenters/organizations.go | 8 +-- backend/internal/pkg/config/config.go | 6 ++ backend/internal/pkg/models/organization.go | 17 +++-- .../interactors/organizations/interactor.go | 33 ++++++++-- .../usecase/repository/cache/cache.go | 66 +++++++++++++++++++ .../repository/organizations/repository.go | 8 +-- 16 files changed, 189 insertions(+), 23 deletions(-) create mode 100644 backend/internal/usecase/repository/cache/cache.go diff --git a/backend/.vscode/launch.json b/backend/.vscode/launch.json index d2ce7b4..7fdb1c5 100644 --- a/backend/.vscode/launch.json +++ b/backend/.vscode/launch.json @@ -21,7 +21,9 @@ "-db-database=blockd", "-db-user=blockd", "-db-secret=blockd", - "-db-enable-tls=false" + "-db-enable-tls=false", + + "-cache-host=localhost:6379" ] } ] diff --git a/backend/Makefile b/backend/Makefile index 2a274e6..bf12ad1 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -31,7 +31,8 @@ run.local: bin.build -db-user=blockd \ -db-secret=blockd \ -db-enable-tls=false \ - -jwt-secret=local_jwt_secret + -jwt-secret=local_jwt_secret \ + -cache-host=localhost:6379 .PHONY: run.debug run.debug: bin.build @@ -44,7 +45,10 @@ run.debug: bin.build -db-database=blockd \ -db-user=blockd \ -db-secret=blockd \ - -db-enable-tls=false + -db-enable-tls=false \ + -jwt-secret=local_jwt_secret \ + -cache-host=localhost:6379 + start.d: sudo systemctl start docker \ No newline at end of file diff --git a/backend/cmd/main.go b/backend/cmd/main.go index d39021f..9da947e 100644 --- a/backend/cmd/main.go +++ b/backend/cmd/main.go @@ -70,6 +70,16 @@ func main() { &cli.BoolFlag{ Name: "db-enable-tls", }, + + &cli.StringFlag{ + Name: "cache-host", + }, + &cli.StringFlag{ + Name: "cache-user", + }, + &cli.StringFlag{ + Name: "cache-secret", + }, }, Action: func(c *cli.Context) error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) @@ -93,6 +103,10 @@ func main() { Database: c.String("db-database"), User: c.String("db-user"), Secret: c.String("db-secret"), + + CacheHost: c.String("cache-host"), + CacheUser: c.String("cache-user"), + CacheSecret: c.String("cache-secret"), }, } diff --git a/backend/docker-compose.yaml b/backend/docker-compose.yaml index c931bf5..88de4e0 100644 --- a/backend/docker-compose.yaml +++ b/backend/docker-compose.yaml @@ -44,6 +44,16 @@ services: start_period: 5s profiles: [blockd, database, noback] + blockd-cache: + container_name: blockd-cache + image: redis:7.2.4 + restart: always + networks: + - blockd-net + ports: + - 6379:6379 + profiles: [blockd, database, noback] + prometheus: image: prom/prometheus container_name: prometheus diff --git a/backend/go.mod b/backend/go.mod index 055a98a..dc561b1 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -35,6 +35,7 @@ require ( github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/ethereum/c-kzg-4844 v1.0.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -49,6 +50,7 @@ require ( github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect + github.com/redis/go-redis/v9 v9.5.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/supranational/blst v0.3.11 // indirect diff --git a/backend/go.sum b/backend/go.sum index 12b0ddd..81f9564 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -113,6 +113,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -332,6 +334,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= +github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/backend/internal/factory/interactors.go b/backend/internal/factory/interactors.go index ad9bf92..0a5ee45 100644 --- a/backend/internal/factory/interactors.go +++ b/backend/internal/factory/interactors.go @@ -7,6 +7,7 @@ import ( "github.com/emochka2007/block-accounting/internal/usecase/interactors/jwt" "github.com/emochka2007/block-accounting/internal/usecase/interactors/organizations" "github.com/emochka2007/block-accounting/internal/usecase/interactors/users" + "github.com/emochka2007/block-accounting/internal/usecase/repository/cache" orepo "github.com/emochka2007/block-accounting/internal/usecase/repository/organizations" urepo "github.com/emochka2007/block-accounting/internal/usecase/repository/users" ) @@ -25,6 +26,7 @@ func provideJWTInteractor(c config.Config, usersInteractor users.UsersInteractor func provideOrganizationsInteractor( log *slog.Logger, orgRepo orepo.Repository, + cache cache.Cache, ) organizations.OrganizationsInteractor { - return organizations.NewOrganizationsInteractor(log, orgRepo) + return organizations.NewOrganizationsInteractor(log, orgRepo, cache) } diff --git a/backend/internal/factory/repositories.go b/backend/internal/factory/repositories.go index 62f7a94..4a8e14e 100644 --- a/backend/internal/factory/repositories.go +++ b/backend/internal/factory/repositories.go @@ -2,9 +2,13 @@ package factory import ( "database/sql" + "log/slog" + "github.com/emochka2007/block-accounting/internal/pkg/config" + "github.com/emochka2007/block-accounting/internal/usecase/repository/cache" "github.com/emochka2007/block-accounting/internal/usecase/repository/organizations" "github.com/emochka2007/block-accounting/internal/usecase/repository/users" + "github.com/redis/go-redis/v9" ) func provideUsersRepository(db *sql.DB) users.Repository { @@ -14,3 +18,20 @@ func provideUsersRepository(db *sql.DB) users.Repository { func provideOrganizationsRepository(db *sql.DB) organizations.Repository { return organizations.NewRepository(db) } + +func provideRedisConnection(c config.Config) (*redis.Client, func()) { + r := redis.NewClient(&redis.Options{ + Addr: c.DB.CacheHost, + Username: c.DB.CacheUser, + Password: c.DB.CacheSecret, + }) + + return r, func() { r.Close() } +} + +func provideRedisCache(c *redis.Client, log *slog.Logger) cache.Cache { + return cache.NewRedisCache( + log.WithGroup("redis-cache"), + c, + ) +} diff --git a/backend/internal/factory/wire.go b/backend/internal/factory/wire.go index 890f4f3..4834ccd 100644 --- a/backend/internal/factory/wire.go +++ b/backend/internal/factory/wire.go @@ -13,7 +13,9 @@ import ( func ProvideService(c config.Config) (service.Service, func(), error) { wire.Build( repository.ProvideDatabaseConnection, + provideRedisConnection, provideLogger, + provideRedisCache, provideUsersRepository, provideUsersInteractor, provideOrganizationsRepository, diff --git a/backend/internal/factory/wire_gen.go b/backend/internal/factory/wire_gen.go index 8101b53..b4dc17b 100644 --- a/backend/internal/factory/wire_gen.go +++ b/backend/internal/factory/wire_gen.go @@ -26,13 +26,16 @@ func ProvideService(c config.Config) (service.Service, func(), error) { authPresenter := provideAuthPresenter(jwtInteractor) authController := provideAuthController(logger, usersInteractor, authPresenter, jwtInteractor) organizationsRepository := provideOrganizationsRepository(db) - organizationsInteractor := provideOrganizationsInteractor(logger, organizationsRepository) + client, cleanup2 := provideRedisConnection(c) + cache := provideRedisCache(client, logger) + organizationsInteractor := provideOrganizationsInteractor(logger, organizationsRepository, cache) organizationsPresenter := provideOrganizationsPresenter() organizationsController := provideOrganizationsController(logger, organizationsInteractor, organizationsPresenter) rootController := provideControllers(logger, authController, organizationsController) server := provideRestServer(logger, rootController, c, jwtInteractor) serviceService := service.NewService(logger, server) return serviceService, func() { + cleanup2() cleanup() }, nil } diff --git a/backend/internal/interface/rest/presenters/organizations.go b/backend/internal/interface/rest/presenters/organizations.go index 6b24f3e..841c297 100644 --- a/backend/internal/interface/rest/presenters/organizations.go +++ b/backend/internal/interface/rest/presenters/organizations.go @@ -10,8 +10,8 @@ import ( type OrganizationsPresenter interface { ResponseCreate(organization *models.Organization) ([]byte, error) - ResponseList(orgs []*models.Organization, nextCursor string) ([]byte, error) - Organizations(orgs []*models.Organization) []domain.Organization + ResponseList(orgs []models.Organization, nextCursor string) ([]byte, error) + Organizations(orgs []models.Organization) []domain.Organization } type organizationsPresenter struct { @@ -40,7 +40,7 @@ func (p *organizationsPresenter) ResponseCreate(o *models.Organization) ([]byte, return out, nil } -func (p *organizationsPresenter) ResponseList(orgs []*models.Organization, nextCursor string) ([]byte, error) { +func (p *organizationsPresenter) ResponseList(orgs []models.Organization, nextCursor string) ([]byte, error) { resp := &domain.ListOrganizationsResponse{ Collection: domain.Collection[domain.Organization]{ Items: p.Organizations(orgs), @@ -59,7 +59,7 @@ func (p *organizationsPresenter) ResponseList(orgs []*models.Organization, nextC return out, nil } -func (p *organizationsPresenter) Organizations(orgs []*models.Organization) []domain.Organization { +func (p *organizationsPresenter) Organizations(orgs []models.Organization) []domain.Organization { out := make([]domain.Organization, len(orgs)) for i, o := range orgs { diff --git a/backend/internal/pkg/config/config.go b/backend/internal/pkg/config/config.go index c2cbd4d..056f75f 100644 --- a/backend/internal/pkg/config/config.go +++ b/backend/internal/pkg/config/config.go @@ -22,11 +22,17 @@ type RestConfig struct { } type DBConfig struct { + // persistent database config Host string EnableSSL bool Database string User string Secret string + + // cache config + CacheHost string + CacheUser string + CacheSecret string } type EthConfig struct { diff --git a/backend/internal/pkg/models/organization.go b/backend/internal/pkg/models/organization.go index bbebb04..414eb4a 100644 --- a/backend/internal/pkg/models/organization.go +++ b/backend/internal/pkg/models/organization.go @@ -1,16 +1,21 @@ package models import ( + "encoding/json" "time" "github.com/google/uuid" ) type Organization struct { - ID uuid.UUID - Name string - Address string - WalletSeed []byte - CreatedAt time.Time - UpdatedAt time.Time + ID uuid.UUID `json:"id"` + Name string `json:"name"` + Address string `json:"addess"` + WalletSeed []byte `json:"wallet_seed"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (i Organization) MarshalBinary() ([]byte, error) { + return json.Marshal(i) } diff --git a/backend/internal/usecase/interactors/organizations/interactor.go b/backend/internal/usecase/interactors/organizations/interactor.go index 20a7862..f956a0f 100644 --- a/backend/internal/usecase/interactors/organizations/interactor.go +++ b/backend/internal/usecase/interactors/organizations/interactor.go @@ -11,7 +11,9 @@ import ( "github.com/emochka2007/block-accounting/internal/pkg/ctxmeta" "github.com/emochka2007/block-accounting/internal/pkg/hdwallet" + "github.com/emochka2007/block-accounting/internal/pkg/logger" "github.com/emochka2007/block-accounting/internal/pkg/models" + "github.com/emochka2007/block-accounting/internal/usecase/repository/cache" "github.com/emochka2007/block-accounting/internal/usecase/repository/organizations" "github.com/google/uuid" ) @@ -49,15 +51,18 @@ type OrganizationsInteractor interface { type organizationsInteractor struct { log *slog.Logger orgRepository organizations.Repository + cache cache.Cache } func NewOrganizationsInteractor( log *slog.Logger, orgRepository organizations.Repository, + cache cache.Cache, ) OrganizationsInteractor { return &organizationsInteractor{ log: log, orgRepository: orgRepository, + cache: cache, } } @@ -96,8 +101,12 @@ func (c *organizationsListCursor) decode(s string) error { } type ListResponse struct { - Organizations []*models.Organization - NextCursor string + Organizations []models.Organization `json:"Organizations"` + NextCursor string `json:"NextCursor"` +} + +func (i ListResponse) MarshalBinary() ([]byte, error) { + return json.Marshal(i) } func (i *organizationsInteractor) List( @@ -117,6 +126,16 @@ func (i *organizationsInteractor) List( params.UserId = user.Id() } + out := new(ListResponse) + + // BUG: empty org set fetched from cache + // if err := i.cache.Get(ctx, params, out); err != nil && errors.Is(err, redis.Nil) { + // i.log.Error("no cache hit!", logger.Err(err)) + // } else { + // i.log.Debug("cache hit!", slog.AnyValue(out)) + // return out, nil + // } + if params.Limit <= 0 || params.Limit > 50 { params.Limit = 50 } @@ -158,10 +177,16 @@ func (i *organizationsInteractor) List( } } - return &ListResponse{ + out = &ListResponse{ Organizations: orgs, NextCursor: nextCursor, - }, nil + } + + if err = i.cache.Cache(ctx, params, *out, time.Hour*1); err != nil { + i.log.Error("error add cache record", logger.Err(err)) + } + + return out, nil } func (i *organizationsInteractor) Create( diff --git a/backend/internal/usecase/repository/cache/cache.go b/backend/internal/usecase/repository/cache/cache.go new file mode 100644 index 0000000..7b5c97d --- /dev/null +++ b/backend/internal/usecase/repository/cache/cache.go @@ -0,0 +1,66 @@ +package cache + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/gob" + "fmt" + "log/slog" + "time" + + "github.com/redis/go-redis/v9" +) + +type Cache interface { + // NOTE: dst MUST be a pointer + Get(ctx context.Context, key any, dst any) error + Cache(ctx context.Context, key any, val any, ttl time.Duration) error +} + +type redisCache struct { + log *slog.Logger + client *redis.Client +} + +func NewRedisCache( + log *slog.Logger, + client *redis.Client, +) Cache { + return &redisCache{ + log: log, + client: client, + } +} + +func (c *redisCache) Get(ctx context.Context, key any, dst any) error { + res := c.client.Get(ctx, c.hashKeyStr(key)) + + if res.Err() != nil { + return fmt.Errorf("error fetch data from cache. %w", res.Err()) + } + + return res.Scan(dst) +} + +func (c *redisCache) Cache(ctx context.Context, k any, v any, ttl time.Duration) error { + res := c.client.Set(ctx, c.hashKeyStr(k), v, ttl) + + if res.Err() != nil { + return fmt.Errorf("error add record to cache. %w", res.Err()) + } + + return nil +} + +func (c *redisCache) hashKey(k any) []byte { + var b bytes.Buffer + + gob.NewEncoder(&b).Encode(k) + + return b.Bytes() +} + +func (c *redisCache) hashKeyStr(k any) string { + return base64.StdEncoding.EncodeToString(c.hashKey(k)) +} diff --git a/backend/internal/usecase/repository/organizations/repository.go b/backend/internal/usecase/repository/organizations/repository.go index a91061c..ca9c877 100644 --- a/backend/internal/usecase/repository/organizations/repository.go +++ b/backend/internal/usecase/repository/organizations/repository.go @@ -31,7 +31,7 @@ type AddParticipantParams struct { type Repository interface { Create(ctx context.Context, org models.Organization) error - Get(ctx context.Context, params GetParams) ([]*models.Organization, error) + Get(ctx context.Context, params GetParams) ([]models.Organization, error) Update(ctx context.Context, org models.Organization) error Delete(ctx context.Context, id uuid.UUID) error AddParticipant(ctx context.Context, params AddParticipantParams) error @@ -86,8 +86,8 @@ func (r *repositorySQL) Create(ctx context.Context, org models.Organization) err return nil } -func (r *repositorySQL) Get(ctx context.Context, params GetParams) ([]*models.Organization, error) { - organizations := make([]*models.Organization, 0, params.Limit) +func (r *repositorySQL) Get(ctx context.Context, params GetParams) ([]models.Organization, error) { + organizations := make([]models.Organization, 0, params.Limit) if err := sqltools.Transaction(ctx, r.db, func(ctx context.Context) (err error) { query := sq.Select( @@ -158,7 +158,7 @@ func (r *repositorySQL) Get(ctx context.Context, params GetParams) ([]*models.Or return fmt.Errorf("error scan row. %w", err) } - organizations = append(organizations, &models.Organization{ + organizations = append(organizations, models.Organization{ ID: id, Name: name, Address: address,