Bug fix: workaround cadence client bug of concurrent map writes (#125)

Author: longquanzheng

CONTRIBUTING.md

@@ -76,6 +76,8 @@ cadence adm cl asa --search_attr_key IwfGlobalWorkflowVersion --search_attr_type
 cadence adm cl asa --search_attr_key IwfExecutingStateIds --search_attr_type 1
 cadence adm cl asa --search_attr_key IwfWorkflowType --search_attr_type 1
 ```
+After registering, it may take [up 60s](https://github.com/uber/cadence/blob/d618e32ac5ea05c411cca08c3e4859e800daa1e0/docker/config_template.yaml#L286) for Cadence to load the new search attributes. If you run the test too early, you may see error:
+ like `"IwfWorkflowType is not a valid search attribute key"`
 4. Go to Cadence http://localhost:8088/domains/default/workflows?range=last-30-days
 
 If you run into any issues with Search Attributes registration, use the below command to check the existing Search attributes:

service/interpreter/workflowImpl.go

@@ -79,12 +79,16 @@ func InterpreterImpl(ctx UnifiedContext, provider WorkflowProvider, input servic
 					)
 					return
 				}
-				defer func() {
+				manualDeferFn := func() {
+					// using defer will cause https://github.com/uber-go/cadence-client/issues/1198 in Cadence
+					// so we use manual defer here...
+					// NOTE: must execute this in every place when return...
+					// TODO be extremely careful in this piece of code, and remove this hack when the bug is fixed in Cadence client
 					err := stateExecutingMgr.completeStates(state)
 					if err != nil {
 						errToFailWf = err
 					}
-				}()
+				}
 
 				stateExeId := stateExeIdMgr.IncAndGetNextExecutionId(state.GetStateId())
 				decision, err := executeState(ctx, provider, state, execution, stateExeId, persistenceManager, interStateChannel)
@@ -111,6 +115,7 @@ func InterpreterImpl(ctx UnifiedContext, provider WorkflowProvider, input servic
 				if !shouldClose && decision.HasNextStates() {
 					currentStates = append(currentStates, decision.GetNextStates()...)
 				}
+				manualDeferFn()
 			})
 		}