Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: Git with refs types refactor #53

Open
wants to merge 21 commits into
base: ipfs-develop
Choose a base branch
from
Open

WIP: Git with refs types refactor #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
ffe4112
Change content address data types
Ericson2314 Jul 2, 2020
0d20c9d
WIP start converting the rest
Ericson2314 Jul 2, 2020
7e2133e
Merge remote-tracking branch 'obsidian/git-with-refs' into git-with-r…
Ericson2314 Jul 6, 2020
f307db3
Merge branch 'ipfs-develop' of github.com:obsidiansystems/nix into gi…
Ericson2314 Jul 6, 2020
4287b89
Allow storings some tree and not just hash for IPFS
Ericson2314 Jul 6, 2020
acbda9c
WIP
Ericson2314 Jul 7, 2020
b6249fb
Add missing header
Ericson2314 Jul 7, 2020
8476d8c
Merge branch 'ipfs-develop' into git-with-refs-strawman
Ericson2314 Jul 8, 2020
842dd4f
Use `IPFSGitTreeNode` typedef after all
Ericson2314 Jul 8, 2020
34f000c
Merge branch 'ipfs-develop' into git-with-refs-strawman
Ericson2314 Jul 8, 2020
e723ced
remove cid commands
Ericson2314 Jul 8, 2020
caac4c4
Make ipfs.cc with bigger definitions
Ericson2314 Jul 9, 2020
17e267d
WIP on store-api
Ericson2314 Jul 9, 2020
2da7fec
Get store-api.cc building
Ericson2314 Jul 9, 2020
82521a9
Merge branch 'ipfs-develop' of github.com:obsidiansystems/nix into gi…
Ericson2314 Jul 9, 2020
4eb4d42
Get ipfs.cc building
Ericson2314 Jul 9, 2020
3cb966a
Tiny step towards getting the IPFS store to compile
Ericson2314 Jul 9, 2020
3366d8c
Merge branch 'ipfs-develop' of github.com:obsidiansystems/nix into gi…
Ericson2314 Jul 9, 2020
76540c9
Everything compiles!
Ericson2314 Jul 9, 2020
3bd223b
Merge branch 'ipfs-develop' of github.com:obsidiansystems/nix into gi…
Ericson2314 Jul 9, 2020
db22255
Merge remote-tracking branch 'obsidian/ipfs-develop' into git-with-re…
Ericson2314 Jul 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 74 additions & 76 deletions src/libstore/content-address.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ std::string renderLegacyContentAddress(LegacyContentAddress ca) {
+ makeFileIngestionPrefix(fsh.method)
+ fsh.hash.to_string(Base32, true);
},
[](IPFSHash ih) {
[](IPFSHash<IPFSGitTreeNode> ih) {
// FIXME do Base-32 for consistency
return "ipfs:" + ih.to_string();
}
Expand Down Expand Up @@ -86,7 +86,7 @@ LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) {
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
};
} else if (prefix == "ipfs") {
auto hash = IPFSHash::from_string(rest);
auto hash = IPFSHash<IPFSGitTreeNode>::from_string(rest);
if (hash.hash.type != htSHA256)
throw Error("This IPFS hash should have type SHA-256: %s", hash.to_string());
return hash;
Expand Down Expand Up @@ -134,10 +134,7 @@ std::string renderContentAddress(ContentAddress ca)
.hash = fsh.hash
}});
},
[](IPFSInfo fsh) {
throw Error("ipfs info not handled");
},
[&](IPFSHash ih) {
[&](IPFSHashWithOptValue<IPFSGitTreeNode> ih) {
result += "ipfs:";
result += ih.to_string();
},
Expand Down Expand Up @@ -214,7 +211,9 @@ ContentAddress parseContentAddress(std::string_view rawCa)
refs,
};
} else if (tag == "ipfs") {
info = IPFSHash::from_string(rest);
info = IPFSHashWithOptValue<IPFSGitTreeNode> {
IPFSHash<IPFSGitTreeNode>::from_string(rest)
};
} else
throw UsageError("content address tag \"%s\" is unrecognized. Recogonized tages are \"text\", \"fixed\", or \"ipfs\"", tag);

Expand All @@ -241,10 +240,10 @@ void to_json(nlohmann::json& j, const LegacyContentAddress & ca) {
{ "hash", foh.hash.to_string(Base32, false) },
};
},
[](IPFSHash ih) {
[](IPFSHash<IPFSGitTreeNode> ih) {
return nlohmann::json {
{ "type", "ipfs" },
{ "hash", ih.to_string() },
{ "hash", ih },
};
},
}, ca);
Expand All @@ -270,97 +269,96 @@ void from_json(const nlohmann::json& j, LegacyContentAddress & ca) {
throw Error("invalid type: %s", type);
}

// f01781114 is the cid prefix for a base16 cbor sha1. This hash
// stores the ContentAddress information.
// Needed until https://github.com/nlohmann/json/pull/2117

void to_json(nlohmann::json& j, const std::optional<LegacyContentAddress> & c) {
if (!c)
j = nullptr;
else
to_json(j, *c);
}

void to_json(nlohmann::json& j, const ContentAddress & ca)
void from_json(const nlohmann::json& j, std::optional<LegacyContentAddress> & c) {
if (j.is_null()) {
c = std::nullopt;
} else {
// Dummy value to set tag bit.
c = TextHash { .hash = Hash { htSHA256 } };
from_json(j, *c);
}
}

template<typename T>
void to_json(nlohmann::json& j, const ContentAddressT<T> & c)
{
if (std::holds_alternative<IPFSInfo>(ca.info)) {
auto info = std::get<IPFSInfo>(ca.info);

// FIXME: ipfs sort order is weird, it always puts type before
// references, so we rename it to qtype so it always comes
// before references
j = nlohmann::json {
{ "qtype", "ipfs" },
{ "name", ca.name },
{ "references", info.references },
{ "cid", nlohmann::json {
{ "/", (IPFSHash { .hash = info.hash }).to_string() }
} }
};
} else throw Error("cannot convert to json");
j = nlohmann::json {
{ "name", c.name },
{ "info", c.info },
};
}

void from_json(const nlohmann::json& j, ContentAddress & ca)
template<typename T>
void from_json(const nlohmann::json& j, ContentAddressT<T> & c)
{
std::string_view type = j.at("qtype").get<std::string_view>();
if (type == "ipfs") {
auto cid = j.at("cid").at("/").get<std::string_view>();
ca = ContentAddress {
.name = j.at("name"),
.info = IPFSInfo {
IPFSHash::from_string(cid).hash,
j.at("references").get<PathReferences<IPFSRef>>(),
},
};
} else
throw Error("invalid type: %s", type);
c.name = j.at("name");
from_json(j.at("info"), c.info);
}

void to_json(nlohmann::json& j, const PathReferences<IPFSRef> & references)
template<template <typename> class Ref>
void to_json(nlohmann::json& j, const IPFSGitTreeNodeT<Ref> & c)
{
auto refs = nlohmann::json::array();
for (auto & i : references.references) {
refs.push_back(nlohmann::json {
{ "name", i.name },
{ "cid", nlohmann::json {{ "/", i.hash.to_string() }} }
});
}
// FIXME: ipfs sort order is weird, it always puts type before
// references, so we rename it to qtype so it always comes
Copy link
Collaborator Author

@matthewbauer matthewbauer Jul 9, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's no qtype anymore. Shouldn't we have something telling ipfs what type we have?

// before references
j = nlohmann::json {
{ "gitTree", IPFSHash<void> { c.gitTree } },
{ "zreferences", c.references },
};
}

template<template <typename> class Ref>
void from_json(const nlohmann::json& j, IPFSGitTreeNodeT<Ref> & c)
{
IPFSHash<void> temp {
.hash = Hash { htSHA256 }, // dummy val
};
from_json(j.at("gitTree"), temp);
c.gitTree = std::move(temp.hash);
from_json(j.at("zreferences"), c.references);
}

template<typename T>
void to_json(nlohmann::json& j, const PathReferences<T> & references)
{
// FIXME: ipfs sort order is weird, it always puts references
// before hasSelfReference, so we rename it to zhasSelfReferences
// so it always comes after references

j = nlohmann::json {
{ "zhasSelfReference", references.hasSelfReference },
{ "references", refs }
{ "references", references.references }
};
}

void from_json(const nlohmann::json& j, PathReferences<IPFSRef> & references)
template<typename T>
void from_json(const nlohmann::json& j, PathReferences<T> & references)
{
std::set<IPFSRef> refs;
for (auto & ref : j.at("references")) {
auto cid = ref.at("cid").at("/").get<std::string_view>();
refs.insert(IPFSRef {
.name = std::move(ref.at("name").get<std::string>()),
.hash = IPFSHash::from_string(cid).hash,
});
}
references = PathReferences<IPFSRef> {
std::set<T> refs;
nlohmann::from_json(j.at("references"), refs);
references = PathReferences<T> {
.references = refs,
.hasSelfReference = j.at("zhasSelfReference").get<bool>(),
};
}

// Needed until https://github.com/nlohmann/json/pull/2117
template
void to_json(nlohmann::json& j, const IPFSHash<IPFSGitTreeNode> & c);
template
void from_json(const nlohmann::json& j, IPFSHash<IPFSGitTreeNode> & c);

void to_json(nlohmann::json& j, const std::optional<LegacyContentAddress> & c) {
if (!c)
j = nullptr;
else
to_json(j, *c);
}

void from_json(const nlohmann::json& j, std::optional<LegacyContentAddress> & c) {
if (j.is_null()) {
c = std::nullopt;
} else {
// Dummy value to set tag bit.
c = TextHash { .hash = Hash { htSHA256 } };
from_json(j, *c);
}
}
template
void to_json(nlohmann::json& j, const IPFSGitTreeNode & c);
template
void from_json(const nlohmann::json& j, IPFSGitTreeNode & c);

}
59 changes: 34 additions & 25 deletions src/libstore/content-address.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
#include <nlohmann/json.hpp>
#include <variant>

#include "ipfs.hh"
#include "hash.hh"
#include "ipfs.hh"
#include "path.hh"

namespace nix {
Expand All @@ -31,6 +31,11 @@ struct FixedOutputHash {
std::string printMethodAlgo() const;
};

template<template <typename> class Ref>
struct IPFSGitTreeNodeT;

typedef IPFSGitTreeNodeT<IPFSHashWithOptValue> IPFSGitTreeNode;

/*
We've accumulated several types of content-addressed paths over the years;
fixed-output derivations support multiple hash algorithms and serialisation
Expand All @@ -45,7 +50,7 @@ struct FixedOutputHash {
typedef std::variant<
TextHash, // for paths computed by makeTextPath() / addTextToStore
FixedOutputHash, // for path computed by makeFixedOutputPath
IPFSHash
IPFSHash<IPFSGitTreeNode>
> LegacyContentAddress;

/* Compute the prefix to the hash algorithm which indicates how the files were
Expand Down Expand Up @@ -136,50 +141,54 @@ struct FixedOutputInfo : FixedOutputHash {
PathReferences<StorePath> references;
};

// pair of name and a hash of a content address
struct IPFSRef {
std::string name;
IPFSHash hash;

bool operator < (const IPFSRef & other) const
{
return name < other.name;
// FIXME second field
}
};
template<typename Underlying>
struct ContentAddressT;

struct IPFSInfo {
Hash hash;
template<template <typename> class Ref>
struct IPFSGitTreeNodeT {
Hash gitTree;
// References for the paths
PathReferences<IPFSRef> references;
PathReferences<ContentAddressT<Ref<IPFSGitTreeNodeT<Ref>>>> references;
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why would you ever need more than one level of PathReferences? We can always query the daemon once we have the cid of a reference.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

More importantly, when would we ever fill in more than one layer of references? We handle each path on its own, so no room to share information like this between queryPathInfo calls.

};

// FIXME names
typedef std::variant<
TextInfo,
FixedOutputInfo,
IPFSInfo,
IPFSHash
IPFSHashWithOptValue<IPFSGitTreeNode>
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't this duplicate data because IPFSHashWithOptValue has a hash, but we can also compute the hash from IPFSGitTreeNode?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be better to make IPFSHashWithOptValue a variant of either a hash or a value.

> ContentAddressWithoutName;

struct ContentAddress {
template<typename Underlying>
struct ContentAddressT {
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think a little duplication here would be much better than doing this. What even is ContentAddressT? If you really want to use it, you should call it something like NameWrapper or something.

std::string name;
ContentAddressWithoutName info;
Underlying info;

bool operator < (const ContentAddress & other) const
bool operator < (const ContentAddressT<Underlying> & other) const
{
return name < other.name;
// FIXME second field
}
};

typedef ContentAddressT<ContentAddressWithoutName> ContentAddress;

std::string renderContentAddress(ContentAddress ca);

ContentAddress parseContentAddress(std::string_view rawCa);

void to_json(nlohmann::json& j, const ContentAddress & c);
void from_json(const nlohmann::json& j, ContentAddress & c);
template<template <typename> class Ref>
void to_json(nlohmann::json& j, const IPFSGitTreeNodeT<Ref> & c);
template<template <typename> class Ref>
void from_json(const nlohmann::json& j, IPFSGitTreeNodeT<Ref> & c);

template<typename T>
void to_json(nlohmann::json& j, const ContentAddressT<T> & c);
template<typename T>
void from_json(const nlohmann::json& j, ContentAddressT<T> & c);

void to_json(nlohmann::json& j, const PathReferences<IPFSRef> & c);
void from_json(const nlohmann::json& j, PathReferences<IPFSRef> & c);
template<typename T>
void to_json(nlohmann::json& j, const PathReferences<T> & c);
template<typename T>
void from_json(const nlohmann::json& j, PathReferences<T> & c);

}
40 changes: 16 additions & 24 deletions src/libstore/ipfs-binary-cache-store.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -511,10 +511,8 @@ std::optional<std::string> IPFSBinaryCacheStore::getCidFromCA(ContentAddress ca)
assert(ca_.hash.type == htSHA1);
return "f01781114" + ca_.hash.to_string(Base16, false);
}
} else if (std::holds_alternative<IPFSHash>(ca.info))
return "f01711220" + std::get<IPFSHash>(ca.info).hash.to_string(Base16, false);

assert(!std::holds_alternative<IPFSInfo>(ca.info));
} else if (std::holds_alternative<IPFSHashWithOptValue<IPFSGitTreeNode>>(ca.info))
return "f01711220" + std::get<IPFSHashWithOptValue<IPFSGitTreeNode>>(ca.info).hash.to_string(Base16, false);

return std::nullopt;
}
Expand Down Expand Up @@ -667,7 +665,7 @@ void IPFSBinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSo

return;
}
} else if (info.ca && std::holds_alternative<IPFSHash>(*info.ca)) {
} else if (info.ca && std::holds_alternative<IPFSHash<IPFSGitTreeNode>>(*info.ca)) {
auto nar = make_ref<std::string>(narSource.drain());

AutoDelete tmpDir(createTempDir(), true);
Expand All @@ -677,27 +675,21 @@ void IPFSBinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSo
auto key = ipfsCidFormatBase16(addGit((Path) tmpDir + "/tmp", std::string(info.path.hashPart()), info.hasSelfReference));
assert(std::string(key, 0, 9) == "f01781114");

IPFSInfo caWithRefs { .hash = Hash::parseAny(std::string(key, 9), htSHA1) };
caWithRefs.references.hasSelfReference = info.hasSelfReference;
for (auto & ref : info.references)
caWithRefs.references.references.insert(IPFSRef {
.name = std::string(ref.name()),
.hash = std::get<IPFSHash>(*queryPathInfo(ref)->ca)
});

auto fullCa = *info.fullContentAddressOpt();
auto cid = getCidFromCA(fullCa);

auto realCa = ContentAddress {
.name = std::string(info.path.name()),
.info = caWithRefs
IPFSGitTreeNode caWithRefs {
.gitTree = Hash::parseAny(key.substr(9), htSHA1),
};
caWithRefs.references.hasSelfReference = info.hasSelfReference;
for (auto & ref : info.references) {
caWithRefs.references.references.insert(ContentAddressT<IPFSHashWithOptValue<IPFSGitTreeNode>> {
.name = std::string(ref.name()),
.info = IPFSHashWithOptValue<IPFSGitTreeNode> { IPFSHash<IPFSGitTreeNode> {
.hash = std::get<IPFSHash<IPFSGitTreeNode>>(*queryPathInfo(ref)->ca),
}},
});
}

nlohmann::json json = realCa;

auto cid_ = ipfsCidFormatBase16(std::string(putIpfsDag(realCa, "sha2-256"), 6));
assert(cid_ == cid);
assert(cid_ == "f01711220" + std::get<IPFSHash>(*info.ca).hash.to_string(Base16, false));
auto cid = ipfsCidFormatBase16(std::string(putIpfsDag(caWithRefs, "sha2-256"), 6));
assert(cid == "f01711220" + std::get<IPFSHash<IPFSGitTreeNode>>(*info.ca).hash.to_string(Base16, false));

auto narInfo = make_ref<NarInfo>(info);;
narInfo->narSize = nar->size();
Expand Down
Loading