0187fdd2-f4a4-4c9a-97b4-6604937bf7be

Committed 09 May 2023 12:23AM UTC coverage: 57.253% (-0.002%) from 57.255%

Build # 0187fdd2-f4a4-4c9a-97b4-6604937bf7be

Build Type

buildkite

Committed by David Porter

Commit Message

Merge branch 'master' into feature/zonal-partitioning

Pull Request Pull Request #5252: Feature/zonal partitioning

Run Details

1460 of 1460 new or added lines in 51 files covered. (100.0%)

86909 of 151799 relevant lines covered (57.25%)

2482.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/common/persistence/persistenceErrorInjectionClients.go

// Copyright (c) 2017-2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package persistence

import (
        "context"
        "math/rand"

        "github.com/uber/cadence/common/errors"
        "github.com/uber/cadence/common/log"
        "github.com/uber/cadence/common/log/tag"
)

var (
        // ErrFakeTimeout is a fake persistence timeout error.
        ErrFakeTimeout = &TimeoutError{Msg: "Fake Persistence Timeout Error."}
)

var (
        fakeErrors = []error{
                errors.ErrFakeServiceBusy,
                errors.ErrFakeInternalService,
                ErrFakeTimeout,
                errors.ErrFakeUnhandled,
        }
)

const (
        msgInjectedFakeErr = "Injected fake persistence error"
)

type (
        shardErrorInjectionPersistenceClient struct {
                persistence ShardManager
                errorRate   float64
                logger      log.Logger
        }

        workflowExecutionErrorInjectionPersistenceClient struct {
                persistence ExecutionManager
                errorRate   float64
                logger      log.Logger
        }

        taskErrorInjectionPersistenceClient struct {
                persistence TaskManager
                errorRate   float64
                logger      log.Logger
        }

        historyErrorInjectionPersistenceClient struct {
                persistence HistoryManager
                errorRate   float64
                logger      log.Logger
        }

        metadataErrorInjectionPersistenceClient struct {
                persistence DomainManager
                errorRate   float64
                logger      log.Logger
        }

        visibilityErrorInjectionPersistenceClient struct {
                persistence VisibilityManager
                errorRate   float64
                logger      log.Logger
        }

        queueErrorInjectionPersistenceClient struct {
                persistence QueueManager
                errorRate   float64
                logger      log.Logger
        }

        configStoreErrorInjectionPersistenceClient struct {
                persistence ConfigStoreManager
                errorRate   float64
                logger      log.Logger
        }
)

var _ ShardManager = (*shardErrorInjectionPersistenceClient)(nil)
var _ ExecutionManager = (*workflowExecutionErrorInjectionPersistenceClient)(nil)
var _ TaskManager = (*taskErrorInjectionPersistenceClient)(nil)
var _ HistoryManager = (*historyErrorInjectionPersistenceClient)(nil)
var _ DomainManager = (*metadataErrorInjectionPersistenceClient)(nil)
var _ VisibilityManager = (*visibilityErrorInjectionPersistenceClient)(nil)
var _ QueueManager = (*queueErrorInjectionPersistenceClient)(nil)
var _ ConfigStoreManager = (*configStoreErrorInjectionPersistenceClient)(nil)

// NewShardPersistenceErrorInjectionClient creates an error injection client to manage shards
func NewShardPersistenceErrorInjectionClient(
        persistence ShardManager,
        errorRate float64,
        logger log.Logger,
) ShardManager {
        return &shardErrorInjectionPersistenceClient{
                persistence: persistence,
                errorRate:   errorRate,
                logger:      logger,
        }
}

// NewWorkflowExecutionPersistenceErrorInjectionClient creates an error injection client to manage executions
func NewWorkflowExecutionPersistenceErrorInjectionClient(
        persistence ExecutionManager,
        errorRate float64,
        logger log.Logger,
) ExecutionManager {
        return &workflowExecutionErrorInjectionPersistenceClient{
                persistence: persistence,
                errorRate:   errorRate,
                logger:      logger,
        }
}

// NewTaskPersistenceErrorInjectionClient creates an error injection client to manage tasks
func NewTaskPersistenceErrorInjectionClient(
        persistence TaskManager,
        errorRate float64,
        logger log.Logger,
) TaskManager {
        return &taskErrorInjectionPersistenceClient{
                persistence: persistence,
                errorRate:   errorRate,
                logger:      logger,
        }
}

// NewHistoryPersistenceErrorInjectionClient creates an error injection HistoryManager client to manage workflow execution history
func NewHistoryPersistenceErrorInjectionClient(
        persistence HistoryManager,
        errorRate float64,
        logger log.Logger,
) HistoryManager {
        return &historyErrorInjectionPersistenceClient{
                persistence: persistence,
                errorRate:   errorRate,
                logger:      logger,
        }
}

// NewDomainPersistenceErrorInjectionClient creates an error injection DomainManager client to manage metadata
func NewDomainPersistenceErrorInjectionClient(
        persistence DomainManager,
        errorRate float64,
        logger log.Logger,
) DomainManager {
        return &metadataErrorInjectionPersistenceClient{
                persistence: persistence,
                errorRate:   errorRate,
                logger:      logger,
        }
}

// NewVisibilityPersistenceErrorInjectionClient creates an error injection client to manage visibility
func NewVisibilityPersistenceErrorInjectionClient(
        persistence VisibilityManager,
        errorRate float64,
        logger log.Logger,
) VisibilityManager {
        return &visibilityErrorInjectionPersistenceClient{
                persistence: persistence,
                errorRate:   errorRate,
                logger:      logger,
        }
}

// NewQueuePersistenceErrorInjectionClient creates an error injection client to manage queue
func NewQueuePersistenceErrorInjectionClient(
        persistence QueueManager,
        errorRate float64,
        logger log.Logger,
) QueueManager {
        return &queueErrorInjectionPersistenceClient{
                persistence: persistence,
                errorRate:   errorRate,
                logger:      logger,
        }
}

// NewConfigStoreErrorInjectionPersistenceClient creates an error injection client to manage config store
func NewConfigStoreErrorInjectionPersistenceClient(
        persistence ConfigStoreManager,
        errorRate float64,
        logger log.Logger,
) ConfigStoreManager {
        return &configStoreErrorInjectionPersistenceClient{
                persistence: persistence,
                errorRate:   errorRate,
                logger:      logger,
        }
}

func (p *shardErrorInjectionPersistenceClient) GetName() string {
        return p.persistence.GetName()
}

func (p *shardErrorInjectionPersistenceClient) CreateShard(
        ctx context.Context,
        request *CreateShardRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.CreateShard(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr, tag.StoreOperationCreateShard, tag.Error(fakeErr), tag.Bool(forwardCall), tag.StoreError(persistenceErr))
                return fakeErr
        }
        return persistenceErr
}

func (p *shardErrorInjectionPersistenceClient) GetShard(
        ctx context.Context,
        request *GetShardRequest,
) (*GetShardResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetShardResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetShard(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr, tag.StoreOperationGetShard, tag.Error(fakeErr), tag.Bool(forwardCall), tag.StoreError(persistenceErr))
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *shardErrorInjectionPersistenceClient) UpdateShard(
        ctx context.Context,
        request *UpdateShardRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.UpdateShard(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr, tag.StoreOperationUpdateShard, tag.Error(fakeErr), tag.Bool(forwardCall), tag.StoreError(persistenceErr))
                return fakeErr
        }
        return persistenceErr
}

func (p *shardErrorInjectionPersistenceClient) Close() {
        p.persistence.Close()
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetName() string {
        return p.persistence.GetName()
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetShardID() int {
        return p.persistence.GetShardID()
}

func (p *workflowExecutionErrorInjectionPersistenceClient) CreateWorkflowExecution(
        ctx context.Context,
        request *CreateWorkflowExecutionRequest,
) (*CreateWorkflowExecutionResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *CreateWorkflowExecutionResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.CreateWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCreateWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetWorkflowExecution(
        ctx context.Context,
        request *GetWorkflowExecutionRequest,
) (*GetWorkflowExecutionResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetWorkflowExecutionResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) UpdateWorkflowExecution(
        ctx context.Context,
        request *UpdateWorkflowExecutionRequest,
) (*UpdateWorkflowExecutionResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *UpdateWorkflowExecutionResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.UpdateWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationUpdateWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) ConflictResolveWorkflowExecution(
        ctx context.Context,
        request *ConflictResolveWorkflowExecutionRequest,
) (*ConflictResolveWorkflowExecutionResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ConflictResolveWorkflowExecutionResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ConflictResolveWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationConflictResolveWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) DeleteWorkflowExecution(
        ctx context.Context,
        request *DeleteWorkflowExecutionRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationDeleteWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) DeleteCurrentWorkflowExecution(
        ctx context.Context,
        request *DeleteCurrentWorkflowExecutionRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteCurrentWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationDeleteCurrentWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetCurrentExecution(
        ctx context.Context,
        request *GetCurrentExecutionRequest,
) (*GetCurrentExecutionResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetCurrentExecutionResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetCurrentExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetCurrentExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) ListCurrentExecutions(
        ctx context.Context,
        request *ListCurrentExecutionsRequest,
) (*ListCurrentExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListCurrentExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListCurrentExecutions(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListCurrentExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) IsWorkflowExecutionExists(
        ctx context.Context,
        request *IsWorkflowExecutionExistsRequest,
) (*IsWorkflowExecutionExistsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *IsWorkflowExecutionExistsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.IsWorkflowExecutionExists(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationIsWorkflowExecutionExists,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) ListConcreteExecutions(
        ctx context.Context,
        request *ListConcreteExecutionsRequest,
) (*ListConcreteExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListConcreteExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListConcreteExecutions(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListConcreteExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetTransferTasks(
        ctx context.Context,
        request *GetTransferTasksRequest,
) (*GetTransferTasksResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetTransferTasksResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetTransferTasks(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetTransferTasks,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetCrossClusterTasks(
        ctx context.Context,
        request *GetCrossClusterTasksRequest,
) (*GetCrossClusterTasksResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetCrossClusterTasksResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetCrossClusterTasks(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetTransferTasks,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetReplicationTasks(
        ctx context.Context,
        request *GetReplicationTasksRequest,
) (*GetReplicationTasksResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetReplicationTasksResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetReplicationTasks(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetReplicationTasks,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) CompleteTransferTask(
        ctx context.Context,
        request *CompleteTransferTaskRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.CompleteTransferTask(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCompleteTransferTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) RangeCompleteTransferTask(
        ctx context.Context,
        request *RangeCompleteTransferTaskRequest,
) (*RangeCompleteTransferTaskResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *RangeCompleteTransferTaskResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.RangeCompleteTransferTask(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationRangeCompleteTransferTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) CompleteCrossClusterTask(
        ctx context.Context,
        request *CompleteCrossClusterTaskRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.CompleteCrossClusterTask(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCompleteCrossClusterTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) RangeCompleteCrossClusterTask(
        ctx context.Context,
        request *RangeCompleteCrossClusterTaskRequest,
) (*RangeCompleteCrossClusterTaskResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *RangeCompleteCrossClusterTaskResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.RangeCompleteCrossClusterTask(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationRangeCompleteCrossClusterTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) CompleteReplicationTask(
        ctx context.Context,
        request *CompleteReplicationTaskRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.CompleteReplicationTask(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCompleteReplicationTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) RangeCompleteReplicationTask(
        ctx context.Context,
        request *RangeCompleteReplicationTaskRequest,
) (*RangeCompleteReplicationTaskResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *RangeCompleteReplicationTaskResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.RangeCompleteReplicationTask(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationRangeCompleteReplicationTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) PutReplicationTaskToDLQ(
        ctx context.Context,
        request *PutReplicationTaskToDLQRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.PutReplicationTaskToDLQ(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationPutReplicationTaskToDLQ,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetReplicationTasksFromDLQ(
        ctx context.Context,
        request *GetReplicationTasksFromDLQRequest,
) (*GetReplicationTasksFromDLQResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetReplicationTasksFromDLQResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetReplicationTasksFromDLQ(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetReplicationTasksFromDLQ,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetReplicationDLQSize(
        ctx context.Context,
        request *GetReplicationDLQSizeRequest,
) (*GetReplicationDLQSizeResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetReplicationDLQSizeResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetReplicationDLQSize(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetReplicationDLQSize,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) DeleteReplicationTaskFromDLQ(
        ctx context.Context,
        request *DeleteReplicationTaskFromDLQRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteReplicationTaskFromDLQ(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationDeleteReplicationTaskFromDLQ,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) RangeDeleteReplicationTaskFromDLQ(
        ctx context.Context,
        request *RangeDeleteReplicationTaskFromDLQRequest,
) (*RangeDeleteReplicationTaskFromDLQResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *RangeDeleteReplicationTaskFromDLQResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.RangeDeleteReplicationTaskFromDLQ(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationRangeDeleteReplicationTaskFromDLQ,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) CreateFailoverMarkerTasks(
        ctx context.Context,
        request *CreateFailoverMarkersRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.CreateFailoverMarkerTasks(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCreateFailoverMarkerTasks,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) GetTimerIndexTasks(
        ctx context.Context,
        request *GetTimerIndexTasksRequest,
) (*GetTimerIndexTasksResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetTimerIndexTasksResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetTimerIndexTasks(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetTimerIndexTasks,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) CompleteTimerTask(
        ctx context.Context,
        request *CompleteTimerTaskRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.CompleteTimerTask(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCompleteTimerTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) RangeCompleteTimerTask(
        ctx context.Context,
        request *RangeCompleteTimerTaskRequest,
) (*RangeCompleteTimerTaskResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *RangeCompleteTimerTaskResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.RangeCompleteTimerTask(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationRangeCompleteTimerTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *workflowExecutionErrorInjectionPersistenceClient) Close() {
        p.persistence.Close()
}

func (p *taskErrorInjectionPersistenceClient) GetName() string {
        return p.persistence.GetName()
}

func (p *taskErrorInjectionPersistenceClient) CreateTasks(
        ctx context.Context,
        request *CreateTasksRequest,
) (*CreateTasksResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *CreateTasksResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.CreateTasks(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCreateTasks,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *taskErrorInjectionPersistenceClient) GetTasks(
        ctx context.Context,
        request *GetTasksRequest,
) (*GetTasksResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetTasksResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetTasks(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetTasks,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *taskErrorInjectionPersistenceClient) CompleteTask(
        ctx context.Context,
        request *CompleteTaskRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.CompleteTask(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCompleteTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *taskErrorInjectionPersistenceClient) CompleteTasksLessThan(
        ctx context.Context,
        request *CompleteTasksLessThanRequest,
) (*CompleteTasksLessThanResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *CompleteTasksLessThanResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.CompleteTasksLessThan(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCompleteTasksLessThan,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *taskErrorInjectionPersistenceClient) GetOrphanTasks(
        ctx context.Context,
        request *GetOrphanTasksRequest,
) (*GetOrphanTasksResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetOrphanTasksResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetOrphanTasks(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCompleteTask,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *taskErrorInjectionPersistenceClient) LeaseTaskList(
        ctx context.Context,
        request *LeaseTaskListRequest,
) (*LeaseTaskListResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *LeaseTaskListResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.LeaseTaskList(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationLeaseTaskList,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *taskErrorInjectionPersistenceClient) UpdateTaskList(
        ctx context.Context,
        request *UpdateTaskListRequest,
) (*UpdateTaskListResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *UpdateTaskListResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.UpdateTaskList(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationUpdateTaskList,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *taskErrorInjectionPersistenceClient) ListTaskList(
        ctx context.Context,
        request *ListTaskListRequest,
) (*ListTaskListResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListTaskListResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListTaskList(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListTaskList,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *taskErrorInjectionPersistenceClient) DeleteTaskList(
        ctx context.Context,
        request *DeleteTaskListRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteTaskList(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationDeleteTaskList,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *taskErrorInjectionPersistenceClient) Close() {
        p.persistence.Close()
}

func (p *metadataErrorInjectionPersistenceClient) GetName() string {
        return p.persistence.GetName()
}

func (p *metadataErrorInjectionPersistenceClient) CreateDomain(
        ctx context.Context,
        request *CreateDomainRequest,
) (*CreateDomainResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *CreateDomainResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.CreateDomain(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCreateDomain,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *metadataErrorInjectionPersistenceClient) GetDomain(
        ctx context.Context,
        request *GetDomainRequest,
) (*GetDomainResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetDomainResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetDomain(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetDomain,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *metadataErrorInjectionPersistenceClient) UpdateDomain(
        ctx context.Context,
        request *UpdateDomainRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.UpdateDomain(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationUpdateDomain,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *metadataErrorInjectionPersistenceClient) DeleteDomain(
        ctx context.Context,
        request *DeleteDomainRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteDomain(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationDeleteDomain,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *metadataErrorInjectionPersistenceClient) DeleteDomainByName(
        ctx context.Context,
        request *DeleteDomainByNameRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteDomainByName(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationDeleteDomainByName,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *metadataErrorInjectionPersistenceClient) ListDomains(
        ctx context.Context,
        request *ListDomainsRequest,
) (*ListDomainsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListDomainsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListDomains(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListDomains,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *metadataErrorInjectionPersistenceClient) GetMetadata(
        ctx context.Context,
) (*GetMetadataResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetMetadataResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetMetadata(ctx)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetMetadata,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *metadataErrorInjectionPersistenceClient) Close() {
        p.persistence.Close()
}

func (p *visibilityErrorInjectionPersistenceClient) GetName() string {
        return p.persistence.GetName()
}

func (p *visibilityErrorInjectionPersistenceClient) RecordWorkflowExecutionStarted(
        ctx context.Context,
        request *RecordWorkflowExecutionStartedRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.RecordWorkflowExecutionStarted(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationRecordWorkflowExecutionStarted,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) RecordWorkflowExecutionClosed(
        ctx context.Context,
        request *RecordWorkflowExecutionClosedRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.RecordWorkflowExecutionClosed(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationRecordWorkflowExecutionClosed,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) RecordWorkflowExecutionUninitialized(
        ctx context.Context,
        request *RecordWorkflowExecutionUninitializedRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.RecordWorkflowExecutionUninitialized(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationRecordWorkflowExecutionStarted,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) UpsertWorkflowExecution(
        ctx context.Context,
        request *UpsertWorkflowExecutionRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.UpsertWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationUpsertWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) ListOpenWorkflowExecutions(
        ctx context.Context,
        request *ListWorkflowExecutionsRequest,
) (*ListWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListOpenWorkflowExecutions(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListOpenWorkflowExecutions,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) ListClosedWorkflowExecutions(
        ctx context.Context,
        request *ListWorkflowExecutionsRequest,
) (*ListWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListClosedWorkflowExecutions(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListClosedWorkflowExecutions,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) ListOpenWorkflowExecutionsByType(
        ctx context.Context,
        request *ListWorkflowExecutionsByTypeRequest,
) (*ListWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListOpenWorkflowExecutionsByType(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListOpenWorkflowExecutionsByType,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) ListClosedWorkflowExecutionsByType(
        ctx context.Context,
        request *ListWorkflowExecutionsByTypeRequest,
) (*ListWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListClosedWorkflowExecutionsByType(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListClosedWorkflowExecutionsByType,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) ListOpenWorkflowExecutionsByWorkflowID(
        ctx context.Context,
        request *ListWorkflowExecutionsByWorkflowIDRequest,
) (*ListWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListOpenWorkflowExecutionsByWorkflowID(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListOpenWorkflowExecutionsByWorkflowID,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) ListClosedWorkflowExecutionsByWorkflowID(
        ctx context.Context,
        request *ListWorkflowExecutionsByWorkflowIDRequest,
) (*ListWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListClosedWorkflowExecutionsByWorkflowID(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListClosedWorkflowExecutionsByWorkflowID,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) ListClosedWorkflowExecutionsByStatus(
        ctx context.Context,
        request *ListClosedWorkflowExecutionsByStatusRequest,
) (*ListWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListClosedWorkflowExecutionsByStatus(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListClosedWorkflowExecutionsByStatus,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) GetClosedWorkflowExecution(
        ctx context.Context,
        request *GetClosedWorkflowExecutionRequest,
) (*GetClosedWorkflowExecutionResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetClosedWorkflowExecutionResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetClosedWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetClosedWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) DeleteWorkflowExecution(
        ctx context.Context,
        request *VisibilityDeleteWorkflowExecutionRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationVisibilityDeleteWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) DeleteUninitializedWorkflowExecution(
        ctx context.Context,
        request *VisibilityDeleteWorkflowExecutionRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteUninitializedWorkflowExecution(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationVisibilityDeleteWorkflowExecution,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) ListWorkflowExecutions(
        ctx context.Context,
        request *ListWorkflowExecutionsByQueryRequest,
) (*ListWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ListWorkflowExecutions(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationListWorkflowExecutions,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) ScanWorkflowExecutions(
        ctx context.Context,
        request *ListWorkflowExecutionsByQueryRequest,
) (*ListWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ListWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ScanWorkflowExecutions(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationScanWorkflowExecutions,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) CountWorkflowExecutions(
        ctx context.Context,
        request *CountWorkflowExecutionsRequest,
) (*CountWorkflowExecutionsResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *CountWorkflowExecutionsResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.CountWorkflowExecutions(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationCountWorkflowExecutions,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *visibilityErrorInjectionPersistenceClient) Close() {
        p.persistence.Close()
}

func (p *historyErrorInjectionPersistenceClient) GetName() string {
        return p.persistence.GetName()
}

// AppendHistoryNodes add(or override) a node to a history branch
func (p *historyErrorInjectionPersistenceClient) AppendHistoryNodes(
        ctx context.Context,
        request *AppendHistoryNodesRequest,
) (*AppendHistoryNodesResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *AppendHistoryNodesResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.AppendHistoryNodes(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationAppendHistoryNodes,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

// ReadHistoryBranch returns history node data for a branch
func (p *historyErrorInjectionPersistenceClient) ReadHistoryBranch(
        ctx context.Context,
        request *ReadHistoryBranchRequest,
) (*ReadHistoryBranchResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ReadHistoryBranchResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ReadHistoryBranch(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationReadHistoryBranch,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

// ReadHistoryBranchByBatch returns history node data for a branch
func (p *historyErrorInjectionPersistenceClient) ReadHistoryBranchByBatch(
        ctx context.Context,
        request *ReadHistoryBranchRequest,
) (*ReadHistoryBranchByBatchResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ReadHistoryBranchByBatchResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ReadHistoryBranchByBatch(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationReadHistoryBranchByBatch,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

// ReadHistoryBranchByBatch returns history node data for a branch
func (p *historyErrorInjectionPersistenceClient) ReadRawHistoryBranch(
        ctx context.Context,
        request *ReadHistoryBranchRequest,
) (*ReadRawHistoryBranchResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ReadRawHistoryBranchResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ReadRawHistoryBranch(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationReadRawHistoryBranch,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

// ForkHistoryBranch forks a new branch from a old branch
func (p *historyErrorInjectionPersistenceClient) ForkHistoryBranch(
        ctx context.Context,
        request *ForkHistoryBranchRequest,
) (*ForkHistoryBranchResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *ForkHistoryBranchResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ForkHistoryBranch(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationForkHistoryBranch,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

// DeleteHistoryBranch removes a branch
func (p *historyErrorInjectionPersistenceClient) DeleteHistoryBranch(
        ctx context.Context,
        request *DeleteHistoryBranchRequest,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteHistoryBranch(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationDeleteHistoryBranch,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

// GetHistoryTree returns all branch information of a tree
func (p *historyErrorInjectionPersistenceClient) GetHistoryTree(
        ctx context.Context,
        request *GetHistoryTreeRequest,
) (*GetHistoryTreeResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetHistoryTreeResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetHistoryTree(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetHistoryTree,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *historyErrorInjectionPersistenceClient) GetAllHistoryTreeBranches(
        ctx context.Context,
        request *GetAllHistoryTreeBranchesRequest,
) (*GetAllHistoryTreeBranchesResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *GetAllHistoryTreeBranchesResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetAllHistoryTreeBranches(ctx, request)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetAllHistoryTreeBranches,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *historyErrorInjectionPersistenceClient) Close() {
        p.persistence.Close()
}

func (p *queueErrorInjectionPersistenceClient) EnqueueMessage(
        ctx context.Context,
        message []byte,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.EnqueueMessage(ctx, message)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationEnqueueMessage,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) ReadMessages(
        ctx context.Context,
        lastMessageID int64,
        maxCount int,
) ([]*QueueMessage, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response []*QueueMessage
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.ReadMessages(ctx, lastMessageID, maxCount)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationReadMessages,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) UpdateAckLevel(
        ctx context.Context,
        messageID int64,
        clusterName string,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.UpdateAckLevel(ctx, messageID, clusterName)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationUpdateAckLevel,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) GetAckLevels(
        ctx context.Context,
) (map[string]int64, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response map[string]int64
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetAckLevels(ctx)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetAckLevels,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) DeleteMessagesBefore(
        ctx context.Context,
        messageID int64,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteMessagesBefore(ctx, messageID)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationDeleteMessagesBefore,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) EnqueueMessageToDLQ(
        ctx context.Context,
        message []byte,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.EnqueueMessageToDLQ(ctx, message)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationEnqueueMessageToDLQ,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) ReadMessagesFromDLQ(
        ctx context.Context,
        firstMessageID int64,
        lastMessageID int64,
        pageSize int,
        pageToken []byte,
) ([]*QueueMessage, []byte, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response []*QueueMessage
        var token []byte
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, token, persistenceErr = p.persistence.ReadMessagesFromDLQ(ctx, firstMessageID, lastMessageID, pageSize, pageToken)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationReadMessagesFromDLQ,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, nil, fakeErr
        }
        return response, token, persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) RangeDeleteMessagesFromDLQ(
        ctx context.Context,
        firstMessageID int64,
        lastMessageID int64,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.RangeDeleteMessagesFromDLQ(ctx, firstMessageID, lastMessageID)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationRangeDeleteMessagesFromDLQ,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) UpdateDLQAckLevel(
        ctx context.Context,
        messageID int64,
        clusterName string,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.UpdateDLQAckLevel(ctx, messageID, clusterName)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationUpdateDLQAckLevel,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) GetDLQAckLevels(
        ctx context.Context,
) (map[string]int64, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response map[string]int64
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetDLQAckLevels(ctx)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetDLQAckLevels,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) GetDLQSize(
        ctx context.Context,
) (int64, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response int64
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.GetDLQSize(ctx)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationGetDLQSize,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return 0, fakeErr
        }
        return response, persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) DeleteMessageFromDLQ(
        ctx context.Context,
        messageID int64,
) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.DeleteMessageFromDLQ(ctx, messageID)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationDeleteMessageFromDLQ,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *queueErrorInjectionPersistenceClient) Close() {
        p.persistence.Close()
}

func (p *configStoreErrorInjectionPersistenceClient) FetchDynamicConfig(ctx context.Context, cfgType ConfigType) (*FetchDynamicConfigResponse, error) {
        fakeErr := generateFakeError(p.errorRate)

        var response *FetchDynamicConfigResponse
        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                response, persistenceErr = p.persistence.FetchDynamicConfig(ctx, cfgType)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationFetchDynamicConfig,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return nil, fakeErr
        }
        return response, persistenceErr
}

func (p *configStoreErrorInjectionPersistenceClient) UpdateDynamicConfig(ctx context.Context, request *UpdateDynamicConfigRequest, cfgType ConfigType) error {
        fakeErr := generateFakeError(p.errorRate)

        var persistenceErr error
        var forwardCall bool
        if forwardCall = shouldForwardCallToPersistence(fakeErr); forwardCall {
                persistenceErr = p.persistence.UpdateDynamicConfig(ctx, request, cfgType)
        }

        if fakeErr != nil {
                p.logger.Error(msgInjectedFakeErr,
                        tag.StoreOperationUpdateDynamicConfig,
                        tag.Error(fakeErr),
                        tag.Bool(forwardCall),
                        tag.StoreError(persistenceErr),
                )
                return fakeErr
        }
        return persistenceErr
}

func (p *configStoreErrorInjectionPersistenceClient) Close() {
        p.persistence.Close()
}

func shouldForwardCallToPersistence(
        err error,
) bool {
        if err == nil {
                return true
        }

        if err == ErrFakeTimeout || err == errors.ErrFakeUnhandled {
                // forward the call with 50% chance
                return rand.Intn(2) == 0
        }

        return false
}

func generateFakeError(
        errorRate float64,
) error {
        if rand.Float64() < errorRate {
                return fakeErrors[rand.Intn(len(fakeErrors))]
        }

        return nil
}

uber / cadence / 0187fdd2-f4a4-4c9a-97b4-6604937bf7be

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous