Skip to content

Commit 166bf59

Browse files
adrikaguptaadrikagupta
committed
conflicts
2 parents 6dfe06e + 310f300 commit 166bf59

File tree

9 files changed

+33665
-145
lines changed

9 files changed

+33665
-145
lines changed

‎cdap-export-import/Dockerfile.export

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Dockerfile located inside cdap-export-import/
2+
3+
FROM eclipse-temurin:8-jre-focal
4+
WORKDIR /opt/app
5+
6+
# The path is now relative to the cdap-export-import directory
7+
COPY target/cdap-export-import-*.jar app.jar
8+
9+
ENTRYPOINT ["java", "-jar", "app.jar"]

‎cdap-export-import/pom.xml

Lines changed: 95 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -29,27 +29,40 @@
2929
<name>CDAP Export-Import</name>
3030
<packaging>jar</packaging>
3131

32-
<properties>
33-
<guava.version>31.1-jre</guava.version>
34-
</properties>
32+
33+
<dependencyManagement>
34+
<dependencies>
35+
<dependency>
36+
<groupId>com.google.cloud</groupId>
37+
<artifactId>libraries-bom</artifactId>
38+
<version>26.32.0</version>
39+
<type>pom</type>
40+
<scope>import</scope>
41+
</dependency>
42+
</dependencies>
43+
</dependencyManagement>
3544

3645
<dependencies>
3746
<dependency>
3847
<groupId>junit</groupId>
3948
<artifactId>junit</artifactId>
4049
<scope>test</scope>
4150
</dependency>
51+
<!-- <dependency>-->
52+
<!-- <groupId>io.cdap.cdap</groupId>-->
53+
<!-- <artifactId>cdap-storage-spi</artifactId>-->
54+
<!-- <version>6.12.0-SNAPSHOT</version>-->
55+
<!-- <scope>compile</scope>-->
56+
<!-- </dependency>-->
57+
<!-- <dependency>-->
58+
<!-- <groupId>io.cdap.cdap</groupId>-->
59+
<!-- <artifactId>cdap-proto</artifactId>-->
60+
<!-- <version>6.12.0-SNAPSHOT</version>-->
61+
<!-- <scope>compile</scope>-->
62+
<!-- </dependency>-->
4263
<dependency>
43-
<groupId>io.cdap.cdap</groupId>
44-
<artifactId>cdap-storage-spi</artifactId>
45-
<version>6.12.0-SNAPSHOT</version>
46-
<scope>compile</scope>
47-
</dependency>
48-
<dependency>
49-
<groupId>io.cdap.cdap</groupId>
50-
<artifactId>cdap-proto</artifactId>
51-
<version>6.12.0-SNAPSHOT</version>
52-
<scope>compile</scope>
64+
<groupId>com.google.cloud</groupId>
65+
<artifactId>google-cloud-storage</artifactId>
5366
</dependency>
5467
<dependency>
5568
<groupId>io.cdap.cdap</groupId>
@@ -60,41 +73,84 @@
6073
<dependency>
6174
<groupId>com.google.guava</groupId>
6275
<artifactId>guava</artifactId>
63-
<version>${guava.version}</version>
76+
<version>31.1-jre</version>
6477
</dependency>
6578
</dependencies>
6679

6780
<build>
6881
<plugins>
82+
<!-- THIS PLUGIN CREATES THE SELF-CONTAINED, SHADED JAR -->
6983
<plugin>
7084
<groupId>org.apache.maven.plugins</groupId>
7185
<artifactId>maven-shade-plugin</artifactId>
72-
<version>3.2.4</version> <executions>
73-
<execution>
74-
<phase>package</phase>
75-
<goals>
76-
<goal>shade</goal>
77-
</goals>
78-
<configuration>
79-
<transformers>
80-
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
81-
<mainClass>io.cdap.cdap.ExportJobMain</mainClass>
82-
</transformer>
83-
</transformers>
84-
<filters>
85-
<filter>
86-
<artifact>*:*</artifact>
87-
<excludes>
88-
<exclude>META-INF/*.SF</exclude>
89-
<exclude>META-INF/*.DSA</exclude>
90-
<exclude>META-INF/*.RSA</exclude>
91-
</excludes>
92-
</filter>
93-
</filters>
94-
</configuration>
95-
</execution>
96-
</executions>
86+
<version>3.2.4</version>
87+
<executions>
88+
<execution>
89+
<phase>package</phase>
90+
<goals>
91+
<goal>shade</goal>
92+
</goals>
93+
<configuration>
94+
<createDependencyReducedPom>false</createDependencyReducedPom>
95+
<relocations>
96+
<relocation>
97+
<pattern>com.google.common</pattern>
98+
<shadedPattern>cdap.shaded.com.google.common</shadedPattern>
99+
</relocation>
100+
<relocation>
101+
<pattern>com.google.api</pattern>
102+
<shadedPattern>cdap.shaded.com.google.api</shadedPattern>
103+
</relocation>
104+
<!-- Add other relocations as needed -->
105+
</relocations>
106+
<filters>
107+
<filter>
108+
<artifact>*:*</artifact>
109+
<excludes>
110+
<exclude>META-INF/*.SF</exclude>
111+
<exclude>META-INF/*.DSA</exclude>
112+
<exclude>META-INF/*.RSA</exclude>
113+
</excludes>
114+
</filter>
115+
</filters>
116+
</configuration>
117+
</execution>
118+
</executions>
97119
</plugin>
98120
</plugins>
99121
</build>
122+
123+
<!-- <build>-->
124+
<!-- <plugins>-->
125+
<!-- <plugin>-->
126+
<!-- <groupId>org.apache.maven.plugins</groupId>-->
127+
<!-- <artifactId>maven-shade-plugin</artifactId>-->
128+
<!-- <version>3.2.4</version> <executions>-->
129+
<!-- <execution>-->
130+
<!-- <phase>package</phase>-->
131+
<!-- <goals>-->
132+
<!-- <goal>shade</goal>-->
133+
<!-- </goals>-->
134+
<!-- <configuration>-->
135+
<!-- <transformers>-->
136+
<!-- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">-->
137+
<!-- <mainClass>io.cdap.cdap.ExportJobMain</mainClass>-->
138+
<!-- </transformer>-->
139+
<!-- </transformers>-->
140+
<!-- <filters>-->
141+
<!-- <filter>-->
142+
<!-- <artifact>*:*</artifact>-->
143+
<!-- <excludes>-->
144+
<!-- <exclude>META-INF/*.SF</exclude>-->
145+
<!-- <exclude>META-INF/*.DSA</exclude>-->
146+
<!-- <exclude>META-INF/*.RSA</exclude>-->
147+
<!-- </excludes>-->
148+
<!-- </filter>-->
149+
<!-- </filters>-->
150+
<!-- </configuration>-->
151+
<!-- </execution>-->
152+
<!-- </executions>-->
153+
<!-- </plugin>-->
154+
<!-- </plugins>-->
155+
<!-- </build>-->
100156
</project>

‎cdap-export-import/src/main/java/io/cdap/cdap/ExportJobMain.java

Lines changed: 63 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package io.cdap.cdap;
1717

18+
import com.google.gson.Gson;
1819
import com.google.inject.AbstractModule;
1920
import com.google.inject.Guice;
2021
import com.google.inject.Injector;
@@ -23,60 +24,89 @@
2324
import io.cdap.cdap.api.metrics.MetricsCollectionService;
2425
import io.cdap.cdap.common.conf.CConfiguration;
2526
import io.cdap.cdap.common.guice.ConfigModule;
26-
import io.cdap.cdap.common.guice.DFSLocationModule;
27-
import io.cdap.cdap.common.guice.IOModule;
28-
import io.cdap.cdap.common.guice.InMemoryDiscoveryModule;
29-
import io.cdap.cdap.common.guice.RemoteAuthenticatorModules;
3027
import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService;
31-
import io.cdap.cdap.data.runtime.ConstantTransactionSystemClient;
32-
import io.cdap.cdap.data.runtime.DataSetsModules;
3328
import io.cdap.cdap.data.runtime.StorageModule;
34-
import io.cdap.cdap.data.runtime.SystemDatasetRuntimeModule;
35-
import io.cdap.cdap.security.auth.context.AuthenticationContextModules;
36-
import io.cdap.cdap.security.spi.authorization.AccessEnforcer;
37-
import io.cdap.cdap.security.spi.authorization.NoOpAccessController;
3829
import io.cdap.cdap.store.NamespaceTable;
30+
import java.nio.charset.StandardCharsets;
3931
import java.util.ArrayList;
4032
import java.util.Arrays;
4133
import java.util.List;
4234
import io.cdap.cdap.spi.data.transaction.TransactionRunner;
4335
import io.cdap.cdap.spi.data.transaction.TransactionRunners;
44-
import org.apache.tephra.TransactionSystemClient;
4536
import org.slf4j.Logger;
4637
import org.slf4j.LoggerFactory;
4738
import io.cdap.cdap.proto.NamespaceMeta;
39+
import com.google.cloud.storage.BlobId;
40+
import com.google.cloud.storage.BlobInfo;
41+
import com.google.cloud.storage.Storage;
42+
import com.google.cloud.storage.StorageOptions;
4843

4944
public class ExportJobMain
5045
{
5146
private final static Logger LOG = LoggerFactory.getLogger(ExportJobMain.class);
52-
public void exportNamespaces() {
53-
CConfiguration cConf = CConfiguration.create();
54-
List<Module> modules = new ArrayList<>(Arrays.asList(
55-
new ConfigModule(cConf),
56-
new StorageModule(),
57-
new AbstractModule() {
58-
@Override
59-
protected void configure() {
60-
bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class)
61-
.in(Scopes.SINGLETON);
62-
}
63-
}
64-
));
65-
Injector injector = Guice.createInjector(modules);
66-
TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
47+
private static final Gson GSON = new Gson();
48+
public void exportNamespaces(TransactionRunner transactionRunner, String bucketName, Storage gcsClient) {
6749
LOG.debug("Starting export of namespaces");
68-
TransactionRunners.run(transactionRunner, context -> {
69-
NamespaceTable namespaceTable = new NamespaceTable(context);
70-
List<NamespaceMeta> namespaces = namespaceTable.list();
71-
LOG.debug("Found {} namespaces: {}", namespaces.size(), namespaces);
72-
});
50+
51+
try {
52+
TransactionRunners.run(transactionRunner, context -> {
53+
NamespaceTable namespaceTable = new NamespaceTable(context);
54+
List<NamespaceMeta> namespaces = namespaceTable.list();
55+
LOG.info("Found {} namespaces to export.", namespaces.size());
56+
57+
for (NamespaceMeta namespace : namespaces) {
58+
String namespaceId = namespace.getName();
59+
LOG.debug("Processing namespace '{}'...", namespaceId);
60+
61+
String namespaceJson = GSON.toJson(namespace);
62+
63+
String gcsObjectPath = String.format("cdap/namespaces/%s/namespaceMeta", namespaceId);
64+
65+
// Prepare the object for upload.
66+
BlobId blobId = BlobId.of(bucketName, gcsObjectPath);
67+
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("application/json").build();
68+
69+
// Upload the JSON content as bytes.
70+
gcsClient.create(blobInfo, namespaceJson.getBytes(StandardCharsets.UTF_8));
71+
72+
LOG.info("Successfully exported namespace '{}' to gs://{}/{}",
73+
namespaceId, bucketName, gcsObjectPath);
74+
}
75+
}, Exception.class);
76+
} catch (Exception e) {
77+
LOG.error("Failed to export namespaces due to an unexpected transaction error.", e);
78+
throw new RuntimeException("Namespace export failed", e);
79+
}
7380
LOG.debug("Finished exporting namespaces.");
7481
}
7582
public static void main( String[] args )
7683
{
77-
LOG.debug("Args: {}", args);
84+
LOG.debug("Args: {}", args);
85+
CConfiguration cConf = CConfiguration.create();
86+
List<Module> modules = new ArrayList<>(Arrays.asList(
87+
new ConfigModule(cConf),
88+
new StorageModule(),
89+
new AbstractModule() {
90+
@Override
91+
protected void configure() {
92+
bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class)
93+
.in(Scopes.SINGLETON);
94+
}
95+
}
96+
));
97+
Injector injector = Guice.createInjector(modules);
98+
TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
99+
String gcsBucket = args[0];
100+
Storage gcsClient;
101+
try {
102+
gcsClient = StorageOptions.getDefaultInstance().getService();
103+
LOG.info("Successfully initialized Google Cloud Storage client.");
104+
} catch (Exception e) {
105+
LOG.error("Failed to initialize GCS client. Please check authentication.", e);
106+
return;
107+
}
78108
ExportJobMain exportJob = new ExportJobMain();
79-
exportJob.exportNamespaces();
109+
exportJob.exportNamespaces(transactionRunner, gcsBucket, gcsClient);
80110
System.out.println("Job finished.");
81111
}
82112
}

0 commit comments

Comments
 (0)