From 0e1a0148ba962058dc494f6b2e341fdb75932b26 Mon Sep 17 00:00:00 2001 From: Blake Eggleston Date: Fri, 21 Oct 2022 13:10:08 -0700 Subject: [PATCH 001/329] CEP-15 (C*): Messaging and storage engine integration patch by Blake Eggleston; reviewed by Benedict Elliott Smith, David Capwell for CASSANDRA-17103 --- .build/build-accord.xml | 37 + .build/build-checkstyle.xml | 4 +- .build/build-rat.xml | 1 + .build/build-resolver.xml | 4 +- .build/cassandra-build-deps-template.xml | 5 + .build/cassandra-deps-template.xml | 4 + .../post-checkout/100-update-submodules.sh | 41 + .build/git/git-hooks/post-switch | 1 + .../100-verify-submodules-pushed.sh | 98 + .../git-hooks/pre-push/100-push-submodules.sh | 51 + .build/git/install-git-defaults.sh | 117 + .build/parent-pom-template.xml | 36 + .build/sh/bump-accord.sh | 38 + .build/sh/change-submodule-accord.sh | 25 + .build/sh/change-submodule.sh | 52 + .build/sh/development-switch.sh | 117 + .gitmodules | 4 + CHANGES.txt | 7 + CONTRIBUTING.md | 31 + accord_demo.txt | 19 + build.xml | 28 +- conf/cassandra.yaml | 3 + ide/idea-iml-file.xml | 20 + ide/idea/vcs.xml | 3 +- ide/idea/workspace.xml | 20 +- modules/accord | 1 + pylib/cqlshlib/cqlhandling.py | 8 +- simulator.sh | 88 + src/antlr/Cql.g | 1 + src/antlr/Lexer.g | 5 + src/antlr/Parser.g | 249 +- .../audit/AuditLogEntryCategory.java | 2 +- .../cassandra/audit/AuditLogEntryType.java | 1 + .../cassandra/audit/AuditLogFilter.java | 2 +- .../concurrent/SingleThreadExecutorPlus.java | 1 + .../apache/cassandra/concurrent/Stage.java | 2 +- .../config/CassandraRelevantProperties.java | 5 +- .../org/apache/cassandra/config/Config.java | 36 +- .../cassandra/config/DatabaseDescriptor.java | 54 + .../apache/cassandra/cql3/CQLStatement.java | 10 + .../org/apache/cassandra/cql3/Operation.java | 51 +- .../org/apache/cassandra/cql3/Operations.java | 62 +- .../org/apache/cassandra/cql3/Operator.java | 66 +- .../apache/cassandra/cql3/QueryProcessor.java | 8 +- .../org/apache/cassandra/cql3/ResultSet.java | 5 +- .../cassandra/cql3/UpdateParameters.java | 30 +- .../cql3/conditions/ColumnCondition.java | 204 +- .../restrictions/StatementRestrictions.java | 20 +- .../cassandra/cql3/selection/Selectable.java | 8 +- .../cassandra/cql3/selection/Selection.java | 109 +- .../cassandra/cql3/selection/Selector.java | 2 +- .../cql3/statements/BatchStatement.java | 15 +- .../cql3/statements/CQL3CasRequest.java | 131 +- .../cql3/statements/DeleteStatement.java | 2 +- .../statements/ModificationStatement.java | 130 +- .../cql3/statements/QualifiedStatement.java | 46 + .../cql3/statements/SelectStatement.java | 139 +- .../cql3/statements/TransactionStatement.java | 562 ++++ .../cql3/statements/UpdateStatement.java | 90 +- .../cassandra/cql3/terms/Constants.java | 77 +- .../apache/cassandra/cql3/terms/Lists.java | 2 +- .../org/apache/cassandra/cql3/terms/Maps.java | 2 +- .../cassandra/cql3/terms/UserTypes.java | 2 +- .../cql3/transactions/ConditionStatement.java | 148 + .../cql3/transactions/ReferenceOperation.java | 178 ++ .../cql3/transactions/ReferenceValue.java | 155 + .../cql3/transactions/RowDataReference.java | 405 +++ .../transactions/SelectReferenceSource.java | 52 + .../org/apache/cassandra/db/DeletionTime.java | 2 +- .../cassandra/db/MutableDeletionInfo.java | 10 + .../cassandra/db/PartitionPosition.java | 2 +- .../cassandra/db/RangeTombstoneList.java | 9 + .../db/SinglePartitionReadCommand.java | 21 +- .../apache/cassandra/db/SystemKeyspace.java | 11 +- .../org/apache/cassandra/db/WriteType.java | 7 +- .../db/filter/ClusteringIndexNamesFilter.java | 14 - .../cassandra/db/filter/ColumnFilter.java | 2 +- .../cassandra/db/marshal/AbstractType.java | 2 +- .../db/marshal/ByteArrayAccessor.java | 2 + .../apache/cassandra/db/marshal/ListType.java | 6 +- .../apache/cassandra/db/marshal/MapType.java | 4 +- .../apache/cassandra/db/marshal/SetType.java | 10 +- .../cassandra/db/marshal/TupleType.java | 10 + .../apache/cassandra/db/marshal/UserType.java | 19 +- .../db/partitions/AbstractBTreePartition.java | 4 +- .../db/partitions/FilteredPartition.java | 6 + .../cassandra/db/partitions/Partition.java | 1 + .../db/partitions/PartitionUpdate.java | 29 +- .../cassandra/db/rows/AbstractCell.java | 7 + .../apache/cassandra/db/rows/BTreeRow.java | 14 +- .../apache/cassandra/db/rows/ColumnData.java | 1 + .../cassandra/db/rows/ComplexColumnData.java | 7 + .../org/apache/cassandra/db/rows/Row.java | 2 + .../db/virtual/AccordVirtualTables.java | 84 + .../db/virtual/SystemViewsKeyspace.java | 1 + .../cassandra/dht/AccordBytesSplitter.java | 89 + .../apache/cassandra/dht/AccordSplitter.java | 103 + .../cassandra/dht/ByteOrderedPartitioner.java | 61 +- .../cassandra/dht/ComparableObjectToken.java | 7 + .../apache/cassandra/dht/IPartitioner.java | 8 + .../cassandra/dht/LocalPartitioner.java | 14 + .../cassandra/dht/Murmur3Partitioner.java | 45 + .../dht/OrderPreservingPartitioner.java | 71 +- .../cassandra/dht/RandomPartitioner.java | 36 + .../dht/ReversedLongLocalPartitioner.java | 21 + .../org/apache/cassandra/dht/Splitter.java | 8 +- src/java/org/apache/cassandra/dht/Token.java | 48 + .../io/LocalVersionedSerializer.java | 94 + .../cassandra/io/MessageVersionProvider.java | 24 + .../io/sstable/CQLSSTableWriter.java | 10 +- .../cassandra/io/util/DataOutputPlus.java | 1 - .../metrics/AccordClientRequestMetrics.java | 46 + .../org/apache/cassandra/metrics/Sampler.java | 1 + .../apache/cassandra/net/ForwardingInfo.java | 1 + .../org/apache/cassandra/net/Message.java | 3 +- .../cassandra/net/MessagingService.java | 13 + src/java/org/apache/cassandra/net/Verb.java | 142 +- .../cassandra/schema/ColumnMetadata.java | 37 +- .../org/apache/cassandra/schema/Schema.java | 14 +- .../cassandra/schema/SchemaConstants.java | 3 +- .../cassandra/schema/SchemaKeyspace.java | 2 +- .../cassandra/schema/SchemaProvider.java | 18 + .../org/apache/cassandra/schema/TableId.java | 17 +- .../cassandra/schema/TableMetadata.java | 6 +- .../cassandra/serializers/ListSerializer.java | 30 +- .../cassandra/serializers/SetSerializer.java | 2 +- .../apache/cassandra/service/CASRequest.java | 13 +- .../cassandra/service/StorageProxy.java | 53 +- .../cassandra/service/StorageProxyMBean.java | 3 +- .../cassandra/service/StorageService.java | 11 + .../service/StorageServiceMBean.java | 3 + .../service/accord/AccordCallback.java | 69 + .../service/accord/AccordCommand.java | 824 ++++++ .../service/accord/AccordCommandStore.java | 490 ++++ .../service/accord/AccordCommandStores.java | 73 + .../service/accord/AccordCommandsForKey.java | 433 +++ .../accord/AccordConfigurationService.java | 94 + .../service/accord/AccordKeyspace.java | 814 ++++++ .../service/accord/AccordMessageSink.java | 127 + .../service/accord/AccordObjectSizes.java | 228 ++ .../service/accord/AccordPartialCommand.java | 209 ++ .../accord/AccordSerializerVersion.java | 114 + .../service/accord/AccordSerializers.java | 249 ++ .../service/accord/AccordService.java | 322 +++ .../cassandra/service/accord/AccordState.java | 105 + .../service/accord/AccordStateCache.java | 647 +++++ .../service/accord/AccordTopologyUtils.java | 100 + .../service/accord/AccordVerbHandler.java | 48 + .../service/accord/EndpointMapping.java | 136 + .../service/accord/IAccordService.java | 46 + .../service/accord/ListenerProxy.java | 275 ++ .../cassandra/service/accord/ReadFuture.java | 304 ++ .../cassandra/service/accord/TokenRange.java | 82 + .../service/accord/api/AccordAgent.java | 63 + .../service/accord/api/AccordRoutableKey.java | 82 + .../service/accord/api/AccordRoutingKey.java | 334 +++ .../service/accord/api/AccordScheduler.java | 97 + .../service/accord/api/PartitionKey.java | 181 ++ .../service/accord/async/AsyncContext.java | 116 + .../service/accord/async/AsyncLoader.java | 240 ++ .../service/accord/async/AsyncOperation.java | 270 ++ .../service/accord/async/AsyncWriter.java | 334 +++ .../accord/serializers/AcceptSerializers.java | 162 ++ .../accord/serializers/ApplySerializers.java | 89 + .../BeginInvalidationSerializers.java | 103 + .../serializers/CheckStatusSerializers.java | 185 ++ .../serializers/CommandSerializers.java | 200 ++ .../accord/serializers/CommitSerializers.java | 103 + .../accord/serializers/DepsSerializer.java | 200 ++ .../accord/serializers/EnumSerializer.java | 58 + .../serializers/GetDepsSerializers.java | 84 + .../serializers/InformDurableSerializers.java | 58 + .../InformHomeDurableSerializers.java | 68 + .../serializers/InformOfTxnIdSerializers.java | 53 + .../accord/serializers/KeySerializers.java | 401 +++ .../serializers/PreacceptSerializers.java | 115 + .../serializers/ReadDataSerializers.java | 107 + .../serializers/RecoverySerializers.java | 168 ++ .../serializers/TopologySerializers.java | 72 + .../serializers/TxnRequestSerializer.java | 105 + .../serializers/WaitOnCommitSerializer.java | 77 + .../accord/store/AbstractStoredField.java | 152 + .../service/accord/store/StoredBoolean.java | 85 + .../service/accord/store/StoredLong.java | 86 + .../accord/store/StoredNavigableMap.java | 224 ++ .../service/accord/store/StoredSet.java | 249 ++ .../service/accord/store/StoredValue.java | 128 + .../service/accord/txn/AbstractKeySorted.java | 155 + .../accord/txn/AbstractSerialized.java | 83 + .../accord/txn/AccordUpdateParameters.java | 83 + .../service/accord/txn/TxnCondition.java | 584 ++++ .../cassandra/service/accord/txn/TxnData.java | 199 ++ .../service/accord/txn/TxnDataName.java | 257 ++ .../service/accord/txn/TxnNamedRead.java | 173 ++ .../service/accord/txn/TxnQuery.java | 137 + .../cassandra/service/accord/txn/TxnRead.java | 217 ++ .../service/accord/txn/TxnReference.java | 326 +++ .../accord/txn/TxnReferenceOperation.java | 302 ++ .../accord/txn/TxnReferenceOperations.java | 132 + .../service/accord/txn/TxnReferenceValue.java | 215 ++ .../service/accord/txn/TxnUpdate.java | 304 ++ .../service/accord/txn/TxnWrite.java | 395 +++ .../apache/cassandra/service/paxos/Paxos.java | 17 +- .../cassandra/service/paxos/PaxosPrepare.java | 13 +- .../service/paxos/PaxosPrepareRefresh.java | 8 +- .../cassandra/service/paxos/PaxosRepair.java | 33 +- .../cassandra/service/paxos/PaxosState.java | 27 +- .../org/apache/cassandra/tools/NodeTool.java | 1 + .../tools/nodetool/CreateEpochUnsafe.java | 33 + .../cassandra/utils/ArraySerializers.java | 55 + .../cassandra/utils/ByteBufferUtil.java | 112 +- ...alizer.java => CollectionSerializers.java} | 73 +- src/java/org/apache/cassandra/utils/Hex.java | 13 +- .../cassandra/utils/NullableSerializer.java | 14 +- .../cassandra/utils/btree/BTreeSet.java | 11 + .../utils/concurrent/FutureCombiner.java | 1 + .../cassandra/utils/concurrent/Semaphore.java | 1 + .../utils/logging/ClassNameFilter.java | 47 + .../cassandra/cql3/reserved_keywords.txt | 5 + test/conf/cassandra.yaml | 1 + test/conf/logback-dtest.xml | 5 + test/conf/logback-simulator.xml | 24 +- .../apache/cassandra/distributed/api/Row.java | 160 ++ .../distributed/api/SimpleQueryResult.java | 13 +- .../cassandra/distributed/impl/Instance.java | 3 + .../distributed/impl/InstanceConfig.java | 4 +- .../distributed/impl/MessageImpl.java | 1 + .../cassandra/distributed/impl/Query.java | 14 +- .../distributed/impl/UnsafeGossipHelper.java | 1 + .../shared/VersionedApplicationState.java | 1 + .../test/accord/AccordCQLTest.java | 2521 +++++++++++++++++ .../test/accord/AccordFeatureFlagTest.java | 103 + .../test/accord/AccordIntegrationTest.java | 117 + .../test/accord/AccordTestBase.java | 308 ++ .../test/accord/AccordTopologyTest.java | 42 + .../test/jmx/JMXGetterCheckTest.java | 3 +- .../distributed/util/QueryResultUtil.java | 72 +- .../fuzz/topology/HarryTopologyMixupTest.java | 12 +- .../fuzz/topology/TopologyMixupTestBase.java | 22 +- .../simulator/asm/InterceptClasses.java | 4 +- .../cassandra/simulator/ActionList.java | 4 + .../cassandra/simulator/ActionSchedule.java | 12 +- .../simulator/ClusterSimulation.java | 3 + .../org/apache/cassandra/simulator/Debug.java | 32 +- .../cassandra/simulator/SimulationRunner.java | 21 +- .../cassandra/simulator/SimulatorUtils.java | 7 +- .../simulator/cluster/KeyspaceActions.java | 12 +- .../simulator/logging/RunStartDefiner.java | 37 + .../simulator/logging/SeedDefiner.java | 42 + ...bstractPairOfSequencesPaxosSimulation.java | 299 ++ .../paxos/AccordClusterSimulation.java | 87 + .../paxos/AccordSimulationRunner.java | 78 + .../simulator/paxos/HistoryChecker.java | 32 +- .../simulator/paxos/HistoryValidator.java | 52 + .../paxos/LinearizabilityValidator.java | 83 + .../paxos/LoggingHistoryValidator.java | 73 + .../simulator/paxos/Observation.java | 16 +- .../PairOfSequencesAccordSimulation.java | 304 ++ .../paxos/PairOfSequencesPaxosSimulation.java | 226 +- .../simulator/paxos/PaxosSimulation.java | 42 +- .../paxos/StrictSerializabilityValidator.java | 111 + .../systems/InterceptingGlobalMethods.java | 6 +- .../systems/InterceptingMonitors.java | 16 +- .../systems/InterceptorOfGlobalMethods.java | 16 +- .../simulator/systems/SimulatedAction.java | 3 +- .../simulator/systems/SimulatedQuery.java | 5 +- .../simulator/paxos/HistoryValidatorTest.java | 593 ++++ .../test/ShortAccordSimulationTest.java | 34 + .../test/SimulatedVisitExectuor.java | 7 +- .../{utils => utilsfork}/DefaultRandom.java | 2 +- .../unit/accord/{utils => utilsfork}/Gen.java | 32 +- .../accord/{utils => utilsfork}/Gens.java | 26 +- .../{utils => utilsfork}/Invariants.java | 2 +- .../accord/{utils => utilsfork}/Property.java | 78 +- .../{utils => utilsfork}/RandomSource.java | 8 +- .../{utils => utilsfork}/SeedProvider.java | 2 +- .../WrappedRandomSource.java | 4 +- .../async/TimeoutUtils.java | 2 +- test/unit/org/apache/cassandra/Util.java | 8 + .../cassandra/audit/AuditLoggerTest.java | 22 + .../auth/AllowAllCIDRAuthorizerTest.java | 12 +- .../apache/cassandra/auth/AuthTestUtils.java | 24 + .../auth/CIDRGroupsMappingManagerTest.java | 14 +- ...assandraCIDRAuthorizerEnforceModeTest.java | 12 +- ...assandraCIDRAuthorizerMonitorModeTest.java | 14 +- .../auth/CassandraNetworkAuthorizerTest.java | 12 +- .../apache/cassandra/auth/TxnAuthTest.java | 174 ++ .../concurrent/SimulatedExecutorFactory.java | 4 +- .../config/DatabaseDescriptorRefTest.java | 1 + .../config/DatabaseDescriptorTest.java | 2 + .../org/apache/cassandra/cql3/CQLTester.java | 36 +- .../cql3/NodeLocalConsistencyTest.java | 23 +- .../cql3/PreparedStatementsTest.java | 257 +- .../cql3/conditions/ColumnConditionTest.java | 225 ++ .../statements/DescribeStatementTest.java | 2 + .../statements/TransactionStatementTest.java | 372 +++ .../cql3/statements/TxnDataNameTest.java | 69 + .../cassandra/cql3/terms/ListsTest.java | 1 - .../validation/operations/InsertTest.java | 2 +- .../CIDRFilteringMetricsTableTest.java | 15 +- .../dht/ByteOrderedPartitionerTest.java | 16 + .../cassandra/dht/KeyCollisionTest.java | 6 + .../cassandra/dht/LengthPartitioner.java | 44 +- .../dht/OrderPreservingPartitionerTest.java | 18 +- .../cassandra/dht/PartitionerTestCase.java | 114 + .../org/apache/cassandra/index/StubIndex.java | 8 +- .../sai/cql/AbstractSimpleEqTestBase.java | 6 +- .../index/sai/cql/AllTypesSimpleEqTest.java | 4 +- .../cassandra/io/IVersionedSerializers.java | 39 + .../cassandra/net/MessageDeliveryTest.java | 4 +- .../net/SimulatedMessageDelivery.java | 4 +- .../ConcurrentIrWithPreviewFuzzTest.java | 6 +- .../cassandra/repair/FailedAckTest.java | 6 +- .../repair/FailingRepairFuzzTest.java | 6 +- .../apache/cassandra/repair/FuzzTestBase.java | 8 +- .../cassandra/repair/HappyPathFuzzTest.java | 6 +- .../cassandra/repair/SlowMessageFuzzTest.java | 6 +- .../cassandra/schema/MemtableParamsTest.java | 4 +- .../cassandra/schema/ValidationTest.java | 50 +- .../serializers/SerializationUtils.java | 5 - .../accord/AccordCommandStoreTest.java | 186 ++ .../service/accord/AccordCommandTest.java | 193 ++ .../service/accord/AccordStateCacheTest.java | 494 ++++ .../service/accord/AccordTestUtils.java | 273 ++ .../service/accord/AccordTopologyTest.java | 71 + .../service/accord/EndpointMappingTest.java | 42 + .../service/accord/api/AccordKeyTest.java | 133 + .../service/accord/async/AsyncLoaderTest.java | 326 +++ .../accord/async/AsyncOperationTest.java | 225 ++ .../service/accord/async/AsyncWriterTest.java | 241 ++ .../serializers/CommandSerializersTest.java | 61 + .../serializers/TopologySerializersTest.java | 40 + .../service/accord/store/StoredMapTest.java | 203 ++ .../service/accord/store/StoredSetTest.java | 202 ++ .../service/accord/store/StoredValueTest.java | 85 + .../accord/txn/AbstractKeySortedTest.java | 158 ++ .../service/accord/txn/TxnUpdateTest.java | 50 + .../utils/AbstractTypeGenerators.java | 43 +- .../cassandra/utils/AssertionUtils.java | 46 +- .../cassandra/utils/CassandraGenerators.java | 12 + .../cassandra/utils/ConfigGenBuilder.java | 6 +- .../cassandra/utils/ConfigGenBuilderTest.java | 4 +- .../apache/cassandra/utils/Generators.java | 59 +- .../cassandra/utils/SerializerTestUtils.java | 74 + .../io/sstable/StressCQLSSTableWriter.java | 1 - 345 files changed, 29879 insertions(+), 991 deletions(-) create mode 100644 .build/build-accord.xml create mode 100755 .build/git/git-hooks/post-checkout/100-update-submodules.sh create mode 120000 .build/git/git-hooks/post-switch create mode 100755 .build/git/git-hooks/pre-commit/100-verify-submodules-pushed.sh create mode 100755 .build/git/git-hooks/pre-push/100-push-submodules.sh create mode 100755 .build/git/install-git-defaults.sh create mode 100755 .build/sh/bump-accord.sh create mode 100755 .build/sh/change-submodule-accord.sh create mode 100755 .build/sh/change-submodule.sh create mode 100755 .build/sh/development-switch.sh create mode 100644 .gitmodules create mode 100644 accord_demo.txt create mode 160000 modules/accord create mode 100755 simulator.sh create mode 100644 src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java create mode 100644 src/java/org/apache/cassandra/cql3/transactions/ConditionStatement.java create mode 100644 src/java/org/apache/cassandra/cql3/transactions/ReferenceOperation.java create mode 100644 src/java/org/apache/cassandra/cql3/transactions/ReferenceValue.java create mode 100644 src/java/org/apache/cassandra/cql3/transactions/RowDataReference.java create mode 100644 src/java/org/apache/cassandra/cql3/transactions/SelectReferenceSource.java create mode 100644 src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java create mode 100644 src/java/org/apache/cassandra/dht/AccordBytesSplitter.java create mode 100644 src/java/org/apache/cassandra/dht/AccordSplitter.java create mode 100644 src/java/org/apache/cassandra/io/LocalVersionedSerializer.java create mode 100644 src/java/org/apache/cassandra/io/MessageVersionProvider.java create mode 100644 src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordCallback.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordCommand.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordCommandStore.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordCommandStores.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordCommandsForKey.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordKeyspace.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordMessageSink.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordPartialCommand.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordSerializerVersion.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordService.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordState.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordStateCache.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java create mode 100644 src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java create mode 100644 src/java/org/apache/cassandra/service/accord/EndpointMapping.java create mode 100644 src/java/org/apache/cassandra/service/accord/IAccordService.java create mode 100644 src/java/org/apache/cassandra/service/accord/ListenerProxy.java create mode 100644 src/java/org/apache/cassandra/service/accord/ReadFuture.java create mode 100644 src/java/org/apache/cassandra/service/accord/TokenRange.java create mode 100644 src/java/org/apache/cassandra/service/accord/api/AccordAgent.java create mode 100644 src/java/org/apache/cassandra/service/accord/api/AccordRoutableKey.java create mode 100644 src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java create mode 100644 src/java/org/apache/cassandra/service/accord/api/AccordScheduler.java create mode 100644 src/java/org/apache/cassandra/service/accord/api/PartitionKey.java create mode 100644 src/java/org/apache/cassandra/service/accord/async/AsyncContext.java create mode 100644 src/java/org/apache/cassandra/service/accord/async/AsyncLoader.java create mode 100644 src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java create mode 100644 src/java/org/apache/cassandra/service/accord/async/AsyncWriter.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/AcceptSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/ApplySerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/BeginInvalidationSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/CheckStatusSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/CommitSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/DepsSerializer.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/EnumSerializer.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/GetDepsSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/InformDurableSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/InformHomeDurableSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/InformOfTxnIdSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/PreacceptSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/RecoverySerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/TxnRequestSerializer.java create mode 100644 src/java/org/apache/cassandra/service/accord/serializers/WaitOnCommitSerializer.java create mode 100644 src/java/org/apache/cassandra/service/accord/store/AbstractStoredField.java create mode 100644 src/java/org/apache/cassandra/service/accord/store/StoredBoolean.java create mode 100644 src/java/org/apache/cassandra/service/accord/store/StoredLong.java create mode 100644 src/java/org/apache/cassandra/service/accord/store/StoredNavigableMap.java create mode 100644 src/java/org/apache/cassandra/service/accord/store/StoredSet.java create mode 100644 src/java/org/apache/cassandra/service/accord/store/StoredValue.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/AbstractKeySorted.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/AbstractSerialized.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/AccordUpdateParameters.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnCondition.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnData.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnDataName.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnRead.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnReference.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnReferenceOperation.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnReferenceOperations.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnReferenceValue.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java create mode 100644 src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java create mode 100644 src/java/org/apache/cassandra/tools/nodetool/CreateEpochUnsafe.java create mode 100644 src/java/org/apache/cassandra/utils/ArraySerializers.java rename src/java/org/apache/cassandra/utils/{CollectionSerializer.java => CollectionSerializers.java} (51%) create mode 100644 src/java/org/apache/cassandra/utils/logging/ClassNameFilter.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/AccordTopologyTest.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/logging/RunStartDefiner.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/logging/SeedDefiner.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/paxos/AccordClusterSimulation.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/paxos/AccordSimulationRunner.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/paxos/HistoryValidator.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/paxos/LinearizabilityValidator.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/paxos/LoggingHistoryValidator.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesAccordSimulation.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/paxos/StrictSerializabilityValidator.java create mode 100644 test/simulator/test/org/apache/cassandra/simulator/paxos/HistoryValidatorTest.java create mode 100644 test/simulator/test/org/apache/cassandra/simulator/test/ShortAccordSimulationTest.java rename test/unit/accord/{utils => utilsfork}/DefaultRandom.java (98%) rename test/unit/accord/{utils => utilsfork}/Gen.java (83%) rename test/unit/accord/{utils => utilsfork}/Gens.java (97%) rename test/unit/accord/{utils => utilsfork}/Invariants.java (99%) rename test/unit/accord/{utils => utilsfork}/Property.java (89%) rename test/unit/accord/{utils => utilsfork}/RandomSource.java (97%) rename test/unit/accord/{utils => utilsfork}/SeedProvider.java (98%) rename test/unit/accord/{utils => utilsfork}/WrappedRandomSource.java (95%) rename test/unit/accord/{utils => utilsfork}/async/TimeoutUtils.java (98%) create mode 100644 test/unit/org/apache/cassandra/auth/TxnAuthTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/statements/TransactionStatementTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/statements/TxnDataNameTest.java create mode 100644 test/unit/org/apache/cassandra/io/IVersionedSerializers.java create mode 100644 test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/AccordStateCacheTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java create mode 100644 test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/api/AccordKeyTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/async/AsyncWriterTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/serializers/CommandSerializersTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/serializers/TopologySerializersTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/store/StoredMapTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/store/StoredSetTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/store/StoredValueTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/txn/AbstractKeySortedTest.java create mode 100644 test/unit/org/apache/cassandra/service/accord/txn/TxnUpdateTest.java create mode 100644 test/unit/org/apache/cassandra/utils/SerializerTestUtils.java diff --git a/.build/build-accord.xml b/.build/build-accord.xml new file mode 100644 index 000000000000..eeadf4dd1883 --- /dev/null +++ b/.build/build-accord.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/.build/build-checkstyle.xml b/.build/build-checkstyle.xml index af5867e4aa9a..0484e4098c66 100644 --- a/.build/build-checkstyle.xml +++ b/.build/build-checkstyle.xml @@ -19,7 +19,7 @@ - + @@ -45,7 +45,7 @@ - + diff --git a/.build/build-rat.xml b/.build/build-rat.xml index fea028363c07..2fbacb74d0ec 100644 --- a/.build/build-rat.xml +++ b/.build/build-rat.xml @@ -76,6 +76,7 @@ + diff --git a/.build/build-resolver.xml b/.build/build-resolver.xml index 42bcc82512d0..49d1e8ba793d 100644 --- a/.build/build-resolver.xml +++ b/.build/build-resolver.xml @@ -178,7 +178,7 @@ - + @@ -206,7 +206,7 @@ - + diff --git a/.build/cassandra-build-deps-template.xml b/.build/cassandra-build-deps-template.xml index 4ec59cdf2d4b..c6b56955e013 100644 --- a/.build/cassandra-build-deps-template.xml +++ b/.build/cassandra-build-deps-template.xml @@ -155,5 +155,10 @@ org.bouncycastle bcutil-jdk18on + + org.apache.cassandra + cassandra-accord + tests + diff --git a/.build/cassandra-deps-template.xml b/.build/cassandra-deps-template.xml index a7c27ee12666..e6afd9b9b018 100644 --- a/.build/cassandra-deps-template.xml +++ b/.build/cassandra-deps-template.xml @@ -116,6 +116,10 @@ org.mindrot jbcrypt + + org.apache.cassandra + cassandra-accord + io.airlift airline diff --git a/.build/git/git-hooks/post-checkout/100-update-submodules.sh b/.build/git/git-hooks/post-checkout/100-update-submodules.sh new file mode 100755 index 000000000000..b495ed086054 --- /dev/null +++ b/.build/git/git-hooks/post-checkout/100-update-submodules.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Redirect output to stderr. +exec 1>&2 + +#set -o xtrace +set -o errexit +set -o pipefail +set -o nounset + +bin="$(cd "$(dirname "$0")" > /dev/null; pwd)" + +_main() { + # In case the usage happens at a different layer, make sure to cd to the toplevel + local root_dir + root_dir="$(git rev-parse --show-toplevel)" + cd "$root_dir" + + if [[ ! -e .gitmodules ]]; then + # nothing to see here, look away! + return 0 + fi + git submodule update --init --recursive +} + +_main "$@" diff --git a/.build/git/git-hooks/post-switch b/.build/git/git-hooks/post-switch new file mode 120000 index 000000000000..5513d1deed30 --- /dev/null +++ b/.build/git/git-hooks/post-switch @@ -0,0 +1 @@ +post-checkout \ No newline at end of file diff --git a/.build/git/git-hooks/pre-commit/100-verify-submodules-pushed.sh b/.build/git/git-hooks/pre-commit/100-verify-submodules-pushed.sh new file mode 100755 index 000000000000..c54099ac0f9a --- /dev/null +++ b/.build/git/git-hooks/pre-commit/100-verify-submodules-pushed.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +## When working with submodules the top level project (Apache Cassandra) needs to commit all submodule +## changes so the top level knows what SHA to use. When working in a development environment it is +## common that multiple commits will exist in both projects, if the submodule has its history +## rewritten, then historic top level commits are no longer valid unless the SHAs are pushed to a +## remote repo; this is what the script attempts to do, make sure all SHAs added to the +## Apache Cassandra are backed up to a remote repo to make the Cassandra SHA buildable. +## + +# Redirect output to stderr. +exec 1>&2 + + +#set -o xtrace +set -o errexit +set -o pipefail +set -o nounset + +bin="$(cd "$(dirname "$0")" > /dev/null; pwd)" + +_log() { + echo -e "[pre-commit]\t$*" +} + +error() { + _log "$@" 1>&2 + exit 1 +} + +# Status Table +# A Added +# C Copied +# D Deleted +# M Modified +# R Renamed +# T Type Changed (i.e. regular file, symlink, submodule, …<200b>) +# U Unmerged +# X Unknown +# B Broken +_main() { + # In case the usage happens at a different layer, make sure to cd to the toplevel + local root_dir + root_dir="$(git rev-parse --show-toplevel)" + cd "$root_dir" + + [[ ! -e .gitmodules ]] && return 0 + local enabled=$(git config --bool cassandra.pre-commit.verify-submodules.enabled || echo true) + [ "$enabled" == "false" ] && return 0 + local submodules=( $(git config --file .gitmodules --get-regexp path | awk '{ print $2 }') ) + + local is_submodule=false + local git_sub_dir + local git_sha + while read status file; do + is_submodule=false + for to_check in "${submodules[*]}"; do + if [[ "$to_check" == "$file" ]]; then + is_submodule=true + break + fi + done + if $is_submodule; then + local enabled=$(git config --bool cassandra.pre-commit.verify-submodule-${file}.enabled || echo true) + [ "$enabled" == "false" ] && continue + _log "Submodule detected: ${file} with status ${status}; attempting a push" + _log "\tTo disable pushes, run" + _log "\t\tgit config --local cassandra.pre-commit.verify-submodules.enabled false" + _log "\tOr" + _log "\t\tgit config --local cassandra.pre-commit.verify-submodule-${file}.enabled false" + set -x + git_sub_dir="${file}/.git" + branch="$(git config -f .gitmodules "submodule.${file}.branch")" + [[ -z "${branch:-}" ]] && error "Submodule ${file} does not define a branch" + git_sha="$(git --git-dir "${git_sub_dir}" rev-parse HEAD)" + git --git-dir "${git_sub_dir}" fetch origin + git --git-dir "${git_sub_dir}" branch "origin/${branch}" --contains "${git_sha}" || error "Git commit ${git_sha} not found in $(git remote get-url origin) on branch ${branch}" + fi + done < <(git diff --cached --name-status) +} + +_main "$@" diff --git a/.build/git/git-hooks/pre-push/100-push-submodules.sh b/.build/git/git-hooks/pre-push/100-push-submodules.sh new file mode 100755 index 000000000000..c3daa9559748 --- /dev/null +++ b/.build/git/git-hooks/pre-push/100-push-submodules.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Redirect output to stderr. +exec 1>&2 + +#set -o xtrace +set -o errexit +set -o pipefail +set -o nounset + +bin="$(cd "$(dirname "$0")" > /dev/null; pwd)" + +_main() { + # In case the usage happens at a different layer, make sure to cd to the toplevel + local root_dir + root_dir="$(git rev-parse --show-toplevel)" + cd "$root_dir" + + if [[ ! -e .gitmodules ]]; then + # nothing to see here, look away! + return 0 + fi + + local -r cmd=' +branch="$(git rev-parse --abbrev-ref HEAD)" +[[ "$branch" == "HEAD" ]] && exit 0 + +default_remote="$(git config --local --get branch."${branch}".remote || true)" +remote="${default_remote:-origin}" + +git push --atomic "$remote" "$branch" +' + git submodule foreach --recursive "$cmd" +} + +_main "$@" diff --git a/.build/git/install-git-defaults.sh b/.build/git/install-git-defaults.sh new file mode 100755 index 000000000000..00f1dc435dbe --- /dev/null +++ b/.build/git/install-git-defaults.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#set -o xtrace +set -o errexit +set -o pipefail +set -o nounset + +bin="$(cd "$(dirname "$0")" > /dev/null; pwd)" + +install_template_script() { + local -r name="$1" + local -r d_dir="$2" + + cat < "$name" +#!/usr/bin/env bash + +# This script is autogenerated by the Apache Cassandra build; DO NOT CHANGE! +# When this script is not found it will be installed automatically by the build +# If an existing script is found, that script will be reloated under ${d_dir} as 000-original.sh + +# Redirect output to stderr. +exec 1>&2 + +# Find all scripts to run +for path in \$(find "$d_dir" -name '*.sh' | perl -e "print sort{(split '/', \\\$a)[-1] <=> (split '/', \\\$b)[-1]}<>"); do + "\$path" "\$@" +done +EOF + chmod a+x "$name" +} + +install_hook() { + local -r git_dir="$1" + local -r hooks_dir="${git_dir}/hooks" + local -r name="$2" + local -r d_dir="${hooks_dir}/${name}.d" + local -r trigger_on_install=$3 + + mkdir "${d_dir}" &> /dev/null || true + local -r script_name="${hooks_dir}/${name}" + local installed=true + if [[ -e "$script_name" ]]; then + # was the script already installed? + if ! grep "This script is autogenerated by the Apache Cassandra build" "$script_name" &> /dev/null ; then + echo "$script_name found, but was not generated by the Apache Cassandra build; please remove or move to ${d_dir}/000-original.sh; creating and moving to ${d_dir} will cause it to run as expected, but won't conflict with hooks this build adds" 1>&2 + exit 1 + else + installed=false + fi + fi + # install all hooks + cp "$bin"/git-hooks/"${name}"/* "$d_dir"/ + + # install coordinator hook + install_template_script "$script_name" "$d_dir" + if $installed && $trigger_on_install ; then + echo "Running script $script_name" + "$script_name" + fi +} + +_install_hooks() { + local git_dir + # make sure to use --git-common-dir and not --git-dir to support worktrees + git_dir="$(git rev-parse --git-common-dir 2> /dev/null || true)" + if [[ -z "${git_dir:-}" ]]; then + # not in a git repo, noop + return 0 + fi + + # make sure hooks directory exists; does not exist by default for worktrees + mkdir -p "${git_dir}/hooks" &> /dev/null || true + + install_hook "$git_dir" "post-checkout" true + install_hook "$git_dir" "post-switch" false + install_hook "$git_dir" "pre-commit" false + install_hook "$git_dir" "pre-push" false +} + +_git_config_set() { + local -r name="$1" + # only care about rc + git config --local --get "$name" &> /dev/null +} + +_install_configs() { + # when doing pull, this makes sure submodules are updated + _git_config_set submodule.recurse || git config --local submodule.recurse true +} + +_main() { + local git_dir + # make sure to use --git-common-dir and not --git-dir to support worktrees + git_dir="$(git rev-parse --git-common-dir 2> /dev/null || true)" + # not in a git repo, noop + [[ -z "${git_dir:-}" ]] && return 0 + + _install_configs + _install_hooks +} + +_main "$@" diff --git a/.build/parent-pom-template.xml b/.build/parent-pom-template.xml index 0235ae6ea90e..25d4f0ddc01a 100644 --- a/.build/parent-pom-template.xml +++ b/.build/parent-pom-template.xml @@ -715,6 +715,42 @@ jbcrypt 0.4 + + org.apache.cassandra + cassandra-accord + @version@ + + + org.apache.cassandra + cassandra-all + + + + + org.apache.cassandra + cassandra-accord + @version@ + tests + test + + + org.junit.jupiter + junit-jupiter-api + + + org.junit.jupiter + junit-jupiter-engine + + + ch.qos.logback + logback-classic + + + org.apache.cassandra + cassandra-all + + + io.airlift airline diff --git a/.build/sh/bump-accord.sh b/.build/sh/bump-accord.sh new file mode 100755 index 000000000000..43a476f3edfb --- /dev/null +++ b/.build/sh/bump-accord.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#set -o xtrace +set -o errexit +set -o pipefail +set -o nounset + +_main() { + local home + home="$(git rev-parse --show-toplevel)" + cd "$home" + + git submodule status modules/accord + echo "Is this the correct SHA? [y/n; default=y]" + read correct + if [[ "${correct:-y}" != "y" ]]; then + echo "Please update Accord's SHA and try again" + exit 1 + fi + git commit -m "Change Accord to $(cd modules/accord; git log -1 --format='%h: %B')" modules/accord +} + +_main "$@" diff --git a/.build/sh/change-submodule-accord.sh b/.build/sh/change-submodule-accord.sh new file mode 100755 index 000000000000..997db3dc2c29 --- /dev/null +++ b/.build/sh/change-submodule-accord.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#set -o xtrace +set -o errexit +set -o pipefail +set -o nounset + +bin="$(cd "$(dirname "$0")" > /dev/null; pwd)" + +"$bin"/change-submodule.sh modules/accord 'https://github.com/apache/cassandra-accord.git' trunk diff --git a/.build/sh/change-submodule.sh b/.build/sh/change-submodule.sh new file mode 100755 index 000000000000..6ab2d3795afd --- /dev/null +++ b/.build/sh/change-submodule.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#set -o xtrace +set -o errexit +set -o pipefail +set -o nounset + +_usage() { + cat <&2 + exit 1 +} + +_usage() { + cat < +``` + +When changes are made to a submodule (such as to accord), you need to commit and update the reference in Apache Cassandra + +``` +$ (cd modules/accord ; git commit -am 'Saving progress') +$ .build/sh/bump-accord.sh +``` + +## Commit and Merge Process + +Due to the nature of submodules, the changes to the submodules must be committed and pushed before the changes to Apache Cassandra; these are different repositories so git's `--atomic` does not prevent conflicts from concurrent merges; the basic process is as follows: + +* Follow the normal merge process for the submodule +* Update Apache Cassandra's submodule entry to point to the newly committed change; follow the Accord example below for an example + +``` +$ .build/sh/change-submodule-accord.sh +$ .build/sh/bump-accord.sh +``` + # Useful Links - How you can contribute to Apache Cassandra [presentation](http://www.slideshare.net/yukim/cassandrasummit2013) by Yuki Morishita diff --git a/accord_demo.txt b/accord_demo.txt new file mode 100644 index 000000000000..b8834515221c --- /dev/null +++ b/accord_demo.txt @@ -0,0 +1,19 @@ + +ccm create accord-cql-poc -n 3 +ccm start + +bin/cqlsh -e "create keyspace ks with replication={'class':'SimpleStrategy', 'replication_factor':3};" +bin/cqlsh -e "create table ks.tbl1 (k int primary key, v int);" +bin/cqlsh -e "create table ks.tbl2 (k int primary key, v int);" + +bin/nodetool -h 0000:0000:0000:0000:0000:ffff:7f00:0001 -p 7100 createepochunsafe +bin/nodetool -h 0000:0000:0000:0000:0000:ffff:7f00:0001 -p 7200 createepochunsafe +bin/nodetool -h 0000:0000:0000:0000:0000:ffff:7f00:0001 -p 7300 createepochunsafe + +BEGIN TRANSACTION + LET row1 = (SELECT * FROM ks.tbl1 WHERE k = 1); + SELECT row1.v; + IF row1 IS NULL THEN + INSERT INTO ks.tbl1 (k, v) VALUES (1, 2); + END IF +COMMIT TRANSACTION; \ No newline at end of file diff --git a/build.xml b/build.xml index 45af0462603b..579079b477b3 100644 --- a/build.xml +++ b/build.xml @@ -100,6 +100,8 @@ the user specifies the tmp.dir property --> + + @@ -109,8 +111,12 @@ + + + + @@ -396,6 +402,7 @@ + @@ -517,7 +524,8 @@ - + + @@ -961,6 +969,9 @@ + + + @@ -980,6 +991,7 @@ + @@ -2049,6 +2061,7 @@ + + + + + + + diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 546333dfcfd3..26a3b84a6bcc 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1934,6 +1934,9 @@ drop_compact_storage_enabled: false # Whether or not USE is allowed. This is enabled by default to avoid failure on upgrade. #use_statements_enabled: true +# Enables the execution of Accord (multi-key) transactions on this node. +accord_transactions_enabled: false + # When the client triggers a protocol exception or unknown issue (Cassandra bug) we increment # a client metric showing this; this logic will exclude specific subnets from updating these # metrics diff --git a/ide/idea-iml-file.xml b/ide/idea-iml-file.xml index 13e66fa61308..1d189db8d6bc 100644 --- a/ide/idea-iml-file.xml +++ b/ide/idea-iml-file.xml @@ -49,6 +49,16 @@ + + + + + + + + + + @@ -56,6 +66,8 @@ + + @@ -63,12 +75,17 @@ + + + + + @@ -76,6 +93,9 @@ + + + diff --git a/ide/idea/vcs.xml b/ide/idea/vcs.xml index 81872fd3f150..a5367a526e4d 100644 --- a/ide/idea/vcs.xml +++ b/ide/idea/vcs.xml @@ -2,6 +2,7 @@ + - \ No newline at end of file + diff --git a/ide/idea/workspace.xml b/ide/idea/workspace.xml index c5c0e28b963b..7f688b3d9626 100644 --- a/ide/idea/workspace.xml +++ b/ide/idea/workspace.xml @@ -183,24 +183,38 @@