Skip to content

Commit 3525edd

Browse files
olenagerasimovaolenagerasimova
authored
feat: log upload/remove events (#553)
* feat: log upload/remove events * correct optional
1 parent 560d6d6 commit 3525edd

File tree

14 files changed

+514
-105
lines changed

14 files changed

+514
-105
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ SOFTWARE.
101101
<dependency>
102102
<groupId>com.artipie</groupId>
103103
<artifactId>http</artifactId>
104-
<version>v1.2.3</version>
104+
<version>v1.2.18</version>
105105
</dependency>
106106
<dependency>
107107
<groupId>javax.xml.bind</groupId>

‎src/main/java/com/artipie/rpm/asto/AstoMetadataRemove.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import com.artipie.asto.misc.UncheckedIOScalar;
1111
import com.artipie.asto.streams.StorageValuePipeline;
1212
import com.artipie.rpm.RepoConfig;
13+
import com.artipie.rpm.meta.PackageInfo;
1314
import com.artipie.rpm.meta.XmlAlter;
1415
import com.artipie.rpm.meta.XmlMaid;
1516
import com.artipie.rpm.meta.XmlPackage;
@@ -20,6 +21,7 @@
2021
import java.util.ArrayList;
2122
import java.util.Collection;
2223
import java.util.List;
24+
import java.util.Optional;
2325
import java.util.UUID;
2426
import java.util.concurrent.CompletableFuture;
2527
import java.util.concurrent.CompletionStage;
@@ -43,14 +45,31 @@ public final class AstoMetadataRemove {
4345
*/
4446
private final RepoConfig cnfg;
4547

48+
/**
49+
* Collection with removed packages info if required.
50+
*/
51+
private final Optional<Collection<PackageInfo>> infos;
52+
4653
/**
4754
* Ctor.
4855
* @param asto Abstract storage
4956
* @param cnfg Repos config
57+
* @param infos Collection with removed packages info if required
5058
*/
51-
public AstoMetadataRemove(final Storage asto, final RepoConfig cnfg) {
59+
public AstoMetadataRemove(final Storage asto, final RepoConfig cnfg,
60+
final Optional<Collection<PackageInfo>> infos) {
5261
this.asto = asto;
5362
this.cnfg = cnfg;
63+
this.infos = infos;
64+
}
65+
66+
/**
67+
* Ctor.
68+
* @param asto Abstract storage
69+
* @param cnfg Repos config
70+
*/
71+
public AstoMetadataRemove(final Storage asto, final RepoConfig cnfg) {
72+
this(asto, cnfg, Optional.empty());
5473
}
5574

5675
/**
@@ -119,7 +138,7 @@ private CompletionStage<Long> removePackages(
119138
final InputStream input = opt.map(new UncheckedIOFunc<>(GZIPInputStream::new))
120139
.get();
121140
if (pckg == XmlPackage.PRIMARY) {
122-
maid = new XmlPrimaryMaid.Stream(input, out);
141+
maid = new XmlPrimaryMaid.Stream(input, out, this.infos);
123142
} else {
124143
maid = new XmlMaid.ByPkgidAttr.Stream(input, out);
125144
}

‎src/main/java/com/artipie/rpm/asto/AstoRepoAdd.java‎

Lines changed: 54 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import com.artipie.asto.rx.RxStorageWrapper;
1212
import com.artipie.rpm.RepoConfig;
1313
import com.artipie.rpm.http.RpmUpload;
14+
import com.artipie.rpm.meta.PackageInfo;
15+
import com.artipie.rpm.pkg.HeaderTags;
1416
import com.artipie.rpm.pkg.Package;
1517
import com.jcabi.log.Logger;
1618
import hu.akarnokd.rxjava2.interop.SingleInterop;
@@ -19,6 +21,7 @@
1921
import java.util.List;
2022
import java.util.concurrent.CompletableFuture;
2123
import java.util.concurrent.CompletionStage;
24+
import java.util.stream.Collectors;
2225

2326
/**
2427
* Add packages to metadata and repository.
@@ -60,38 +63,60 @@ public AstoRepoAdd(final Storage asto, final RepoConfig cnfg) {
6063
public CompletionStage<Void> perform() {
6164
return this.read().thenCompose(
6265
list -> new AstoMetadataAdd(this.asto, this.cnfg).perform(list)
63-
).thenCompose(
64-
temp -> new AstoCreateRepomd(this.asto, this.cnfg).perform(temp).thenCompose(
65-
nothing -> new AstoMetadataNames(this.asto, this.cnfg).prepareNames(temp)
66-
.thenCompose(
67-
keys -> {
68-
final StorageLock lock = new StorageLock(this.asto, AstoRepoAdd.META);
69-
return lock.acquire()
70-
.thenCompose(ignored -> this.remove(AstoRepoAdd.META))
66+
).thenCompose(this::generateRepomdAndMoveXmls);
67+
}
68+
69+
/**
70+
* Performs whole workflow to add items, listed in {@link com.artipie.rpm.http.RpmUpload#TO_ADD}
71+
* location, to the repository and metadata files. Returns list with info about added
72+
* packages.
73+
* @return Completable action with added packages info list
74+
*/
75+
public CompletionStage<List<PackageInfo>> performWithResult() {
76+
return this.read().thenCompose(
77+
list -> new AstoMetadataAdd(this.asto, this.cnfg).perform(list)
78+
.thenCompose(this::generateRepomdAndMoveXmls)
79+
.thenApply(
80+
nothing -> list.stream()
81+
.map(info -> new PackageInfo(new HeaderTags(info), info.size()))
82+
.collect(Collectors.toList())
83+
)
84+
);
85+
}
86+
87+
/**
88+
* Creates repomd metadata file and moves all other metadata xmls to repository
89+
* with storage lock.
90+
* @param temp Temp location o metadata files
91+
* @return Completable action
92+
*/
93+
private CompletionStage<Void> generateRepomdAndMoveXmls(final Key temp) {
94+
return new AstoCreateRepomd(this.asto, this.cnfg).perform(temp).thenCompose(
95+
nothing -> new AstoMetadataNames(this.asto, this.cnfg).prepareNames(temp).thenCompose(
96+
keys -> {
97+
final StorageLock lock = new StorageLock(this.asto, AstoRepoAdd.META);
98+
return lock.acquire().thenCompose(ignored -> this.remove(AstoRepoAdd.META))
99+
.thenCompose(
100+
ignored -> CompletableFuture.allOf(
101+
keys.entrySet().stream().map(
102+
entry -> this.asto.move(entry.getKey(), entry.getValue())
103+
).toArray(CompletableFuture[]::new)
104+
)
105+
).thenCompose(
106+
ignored -> this.asto.list(RpmUpload.TO_ADD)
71107
.thenCompose(
72-
ignored -> CompletableFuture.allOf(
73-
keys.entrySet().stream().map(
74-
entry -> this.asto
75-
.move(entry.getKey(), entry.getValue())
108+
list -> CompletableFuture.allOf(
109+
list.stream().map(
110+
key -> this.asto.move(
111+
key, AstoRepoAdd.removeTempPart(key)
112+
)
76113
).toArray(CompletableFuture[]::new)
77114
)
78115
)
79-
.thenCompose(
80-
ignored -> this.asto.list(RpmUpload.TO_ADD).thenCompose(
81-
list -> CompletableFuture.allOf(
82-
list.stream().map(
83-
key -> this.asto.move(
84-
key, AstoRepoAdd.removeTempPart(key)
85-
)
86-
).toArray(CompletableFuture[]::new)
87-
)
88-
)
89-
)
90-
.thenCompose(ignored -> lock.release()).thenCompose(
91-
ignored -> this.remove(temp)
92-
);
93-
}
94-
)
116+
)
117+
.thenCompose(ignored -> lock.release())
118+
.thenCompose(ignored -> this.remove(temp));
119+
}
95120
)
96121
);
97122
}
@@ -144,4 +169,5 @@ private CompletableFuture<Void> remove(final Key key) {
144169
private static Key removeTempPart(final Key key) {
145170
return new KeyExcludeFirst(key, RpmUpload.TO_ADD.string());
146171
}
172+
147173
}

‎src/main/java/com/artipie/rpm/asto/AstoRepoRemove.java‎

Lines changed: 55 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@
1212
import com.artipie.asto.rx.RxStorageWrapper;
1313
import com.artipie.rpm.RepoConfig;
1414
import com.artipie.rpm.http.RpmRemove;
15+
import com.artipie.rpm.meta.PackageInfo;
1516
import hu.akarnokd.rxjava2.interop.SingleInterop;
1617
import io.reactivex.Observable;
1718
import io.reactivex.Single;
1819
import java.util.Collection;
1920
import java.util.List;
21+
import java.util.Optional;
2022
import java.util.concurrent.CompletableFuture;
2123
import java.util.concurrent.CompletionStage;
2224

@@ -42,14 +44,42 @@ public final class AstoRepoRemove {
4244
*/
4345
private final RepoConfig cnfg;
4446

47+
/**
48+
* Collection with removed packages info if required.
49+
*/
50+
private final Optional<Collection<PackageInfo>> infos;
51+
4552
/**
4653
* Ctor.
4754
* @param asto Abstract storage
4855
* @param cnfg Repository config
56+
* @param infos Collection with removed packages info if required
4957
*/
50-
public AstoRepoRemove(final Storage asto, final RepoConfig cnfg) {
58+
public AstoRepoRemove(final Storage asto, final RepoConfig cnfg,
59+
final Optional<Collection<PackageInfo>> infos) {
5160
this.asto = asto;
5261
this.cnfg = cnfg;
62+
this.infos = infos;
63+
}
64+
65+
/**
66+
* Ctor.
67+
* @param asto Abstract storage
68+
* @param cnfg Repository config
69+
* @param infos Collection with removed packages info
70+
*/
71+
public AstoRepoRemove(final Storage asto, final RepoConfig cnfg,
72+
final Collection<PackageInfo> infos) {
73+
this(asto, cnfg, Optional.of(infos));
74+
}
75+
76+
/**
77+
* Ctor.
78+
* @param asto Abstract storage
79+
* @param cnfg Repository config
80+
*/
81+
public AstoRepoRemove(final Storage asto, final RepoConfig cnfg) {
82+
this(asto, cnfg, Optional.empty());
5383
}
5484

5585
/**
@@ -60,29 +90,30 @@ public AstoRepoRemove(final Storage asto, final RepoConfig cnfg) {
6090
* @return Completable action
6191
*/
6292
public CompletionStage<Void> perform(final Collection<String> checksums) {
63-
return new AstoMetadataRemove(this.asto, this.cnfg).perform(checksums).thenCompose(
64-
temp -> new AstoCreateRepomd(this.asto, this.cnfg).perform(temp).thenCompose(
65-
nothing -> new AstoMetadataNames(this.asto, this.cnfg).prepareNames(temp)
66-
.thenCompose(
67-
keys -> {
68-
final StorageLock lock =
69-
new StorageLock(this.asto, AstoRepoRemove.META);
70-
return lock.acquire()
71-
.thenCompose(ignored -> this.remove(AstoRepoRemove.META))
72-
.thenCompose(
73-
ignored -> CompletableFuture.allOf(
74-
keys.entrySet().stream().map(
75-
entry ->
76-
this.asto.move(entry.getKey(), entry.getValue())
77-
).toArray(CompletableFuture[]::new)
78-
)
79-
).thenCompose(ignored -> lock.release()).thenCompose(
80-
ignored -> this.remove(temp)
81-
);
82-
}
83-
)
84-
)
85-
);
93+
return new AstoMetadataRemove(this.asto, this.cnfg, this.infos).perform(checksums)
94+
.thenCompose(
95+
temp -> new AstoCreateRepomd(this.asto, this.cnfg).perform(temp).thenCompose(
96+
nothing -> new AstoMetadataNames(this.asto, this.cnfg).prepareNames(temp)
97+
.thenCompose(
98+
keys -> {
99+
final StorageLock lock =
100+
new StorageLock(this.asto, AstoRepoRemove.META);
101+
return lock.acquire()
102+
.thenCompose(ignored -> this.remove(AstoRepoRemove.META))
103+
.thenCompose(
104+
ignored -> CompletableFuture.allOf(
105+
keys.entrySet().stream().map(
106+
entry ->
107+
this.asto.move(entry.getKey(), entry.getValue())
108+
).toArray(CompletableFuture[]::new)
109+
)
110+
).thenCompose(ignored -> lock.release()).thenCompose(
111+
ignored -> this.remove(temp)
112+
);
113+
}
114+
)
115+
)
116+
);
86117
}
87118

88119
/**

‎src/main/java/com/artipie/rpm/http/RpmRemove.java‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,15 @@
1616
import com.artipie.http.rs.RsWithStatus;
1717
import com.artipie.rpm.RepoConfig;
1818
import com.artipie.rpm.asto.AstoRepoRemove;
19+
import com.artipie.rpm.meta.PackageInfo;
20+
import com.artipie.scheduling.ArtifactEvent;
1921
import java.nio.ByteBuffer;
22+
import java.util.ArrayList;
23+
import java.util.Collection;
2024
import java.util.Locale;
2125
import java.util.Map;
2226
import java.util.Optional;
27+
import java.util.Queue;
2328
import java.util.concurrent.CompletableFuture;
2429
import java.util.concurrent.CompletionStage;
2530
import java.util.stream.StreamSupport;
@@ -56,14 +61,22 @@ public final class RpmRemove implements Slice {
5661
*/
5762
private final RepoConfig cnfg;
5863

64+
/**
65+
* Artifact upload/remove events.
66+
*/
67+
private final Optional<Queue<ArtifactEvent>> events;
68+
5969
/**
6070
* Ctor.
6171
* @param asto Asto storage
6272
* @param cnfg Repo config
73+
* @param events Artifact events
6374
*/
64-
public RpmRemove(final Storage asto, final RepoConfig cnfg) {
75+
public RpmRemove(final Storage asto, final RepoConfig cnfg,
76+
final Optional<Queue<ArtifactEvent>> events) {
6577
this.asto = asto;
6678
this.cnfg = cnfg;
79+
this.events = events;
6780
}
6881

6982
@Override
@@ -81,8 +94,26 @@ public Response response(final String line, final Iterable<Map.Entry<String, Str
8194
.completedFuture(RsStatus.ACCEPTED);
8295
if (valid && this.cnfg.mode() == RepoConfig.UpdateMode.UPLOAD
8396
&& !request.skipUpdate()) {
84-
res = new AstoRepoRemove(this.asto, this.cnfg).perform()
85-
.thenApply(ignored -> RsStatus.ACCEPTED);
97+
res = this.events.map(
98+
queue -> {
99+
final Collection<PackageInfo> infos =
100+
new ArrayList<>(1);
101+
return new AstoRepoRemove(this.asto, this.cnfg, infos)
102+
.perform().thenAccept(
103+
nothing -> infos.forEach(
104+
item -> queue.add(
105+
new ArtifactEvent(
106+
RpmUpload.REPO_TYPE,
107+
this.cnfg.name(), item.name(),
108+
item.version()
109+
)
110+
)
111+
)
112+
);
113+
}
114+
).orElseGet(
115+
() -> new AstoRepoRemove(this.asto, this.cnfg).perform()
116+
).thenApply(ignored -> RsStatus.ACCEPTED);
86117
} else if (!valid) {
87118
res = this.asto.delete(temp)
88119
.thenApply(nothing -> RsStatus.BAD_REQUEST);

0 commit comments

Comments
 (0)