15#include "store-dir-config.hh"
19#include <nlohmann/json_fwd.hpp>
55MakeError(SubstError, Error);
59MakeError(BuildError, Error);
60MakeError(InvalidPath, Error);
61MakeError(Unsupported, Error);
62MakeError(SubstituteGone, Error);
63MakeError(SubstituterDisabled, Error);
65MakeError(InvalidStoreReference, Error);
79typedef std::map<std::string, StorePath> OutputPathMap;
82enum CheckSigsFlag :
bool { NoCheckSigs =
false, CheckSigs =
true };
83enum SubstituteFlag :
bool { NoSubstitute =
false, Substitute =
true };
91enum BuildMode : uint8_t { bmNormal, bmRepair, bmCheck };
92enum TrustedFlag :
bool { NotTrusted =
false, Trusted =
true };
98typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
100struct StoreConfig :
public StoreDirConfig
102 using Params = StoreReference::Params;
104 using StoreDirConfig::StoreDirConfig;
106 StoreConfig() =
delete;
108 static StringSet getDefaultSystemFeatures();
110 virtual ~StoreConfig() { }
115 virtual const std::string
name() = 0;
134 const Setting<int> pathInfoCacheSize{
this, 65536,
"path-info-cache-size",
135 "Size of the in-memory store path metadata cache."};
137 const Setting<bool> isTrusted{
this,
false,
"trusted",
139 Whether paths from this store can be used as substitutes
140 even if they are not signed by a key listed in the
141 [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
145 Setting<int> priority{this, 0,
"priority",
147 Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
148 A lower value means a higher priority.
151 Setting<bool> wantMassQuery{this,
false,
"want-mass-query",
153 Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
156 Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
159 Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations.
168class Store :
public std::enable_shared_from_this<Store>,
public virtual StoreConfig
177 std::chrono::time_point<std::chrono::steady_clock>
time_point = std::chrono::steady_clock::now();
182 std::shared_ptr<const ValidPathInfo>
value;
195 return value !=
nullptr;
204 SharedSync<State> state;
206 std::shared_ptr<NarInfoDiskCache> diskCache;
208 Store(
const Params & params);
223 virtual std::string
getUri() = 0;
243 virtual bool isValidPathUncached(
const StorePath &
path);
259 SubstituteFlag maybeSubstitute = NoSubstitute);
272 constexpr static const char * MissingName =
"x";
305 Callback<std::shared_ptr<const Realisation>> callback)
noexcept;
324 virtual bool realisationIsUntrusted(
const Realisation & )
331 virtual void queryPathInfoUncached(
const StorePath &
path,
332 Callback<std::shared_ptr<const ValidPathInfo>> callback)
noexcept = 0;
333 virtual void queryRealisationUncached(
const DrvOutput &,
334 Callback<std::shared_ptr<const Realisation>> callback)
noexcept = 0;
367 Store * evalStore =
nullptr);
405 SubstitutablePathInfos & infos);
411 RepairFlag
repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
417 using PathsSource = std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>>;
424 RepairFlag
repair = NoRepair,
425 CheckSigsFlag checkSigs = CheckSigs);
430 RepairFlag
repair = NoRepair,
431 CheckSigsFlag checkSigs = CheckSigs);
442 std::string_view
name,
445 HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
446 const StorePathSet & references = StorePathSet(),
448 RepairFlag
repair = NoRepair);
456 std::string_view
name,
459 HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
460 const StorePathSet & references = StorePathSet(),
461 std::optional<Hash> expectedCAHash = {});
482 std::string_view
name,
485 HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
486 const StorePathSet & references = StorePathSet(),
487 RepairFlag
repair = NoRepair) = 0;
519 const std::vector<DerivedPath> & paths,
520 BuildMode buildMode = bmNormal,
521 std::shared_ptr<Store> evalStore =
nullptr);
530 const std::vector<DerivedPath> & paths,
531 BuildMode buildMode = bmNormal,
532 std::shared_ptr<Store> evalStore =
nullptr);
570 BuildMode buildMode = bmNormal);
584 { debug(
"not creating temporary root, store doesn't support GC"); }
592 bool showDerivers,
bool showHash);
654 StorePathSet & out,
bool flipDirection =
false,
655 bool includeOutputs =
false,
bool includeDerivers =
false);
658 StorePathSet & out,
bool flipDirection =
false,
659 bool includeOutputs =
false,
bool includeDerivers =
false);
666 virtual void queryMissing(
const std::vector<DerivedPath> & targets,
667 StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
668 uint64_t & downloadSize, uint64_t & narSize);
693 std::atomic<uint64_t> narInfoRead{0};
694 std::atomic<uint64_t> narInfoReadAverted{0};
695 std::atomic<uint64_t> narInfoMissing{0};
696 std::atomic<uint64_t> narInfoWrite{0};
697 std::atomic<uint64_t> pathInfoCacheSize{0};
698 std::atomic<uint64_t> narRead{0};
699 std::atomic<uint64_t> narReadBytes{0};
700 std::atomic<uint64_t> narReadCompressedBytes{0};
701 std::atomic<uint64_t> narWrite{0};
702 std::atomic<uint64_t> narWriteAverted{0};
703 std::atomic<uint64_t> narWriteBytes{0};
704 std::atomic<uint64_t> narWriteCompressedBytes{0};
705 std::atomic<uint64_t> narWriteCompressionTimeMs{0};
708 const Stats & getStats();
714 StorePathSet
exportReferences(
const StorePathSet & storePaths,
const StorePathSet & inputPaths);
729 state.lock()->pathInfoCache.clear();
757 virtual Path toRealPath(
const Path & storePath)
773 virtual std::optional<std::string> getVersion() {
return {}; }
788 throw Unsupported(
"operation '%s' is not supported by store '%s'", op,
getUri());
800 const StorePath & storePath,
801 RepairFlag
repair = NoRepair,
802 CheckSigsFlag checkSigs = CheckSigs);
813std::map<StorePath, StorePath> copyPaths(
815 const std::set<RealisedPath> &,
816 RepairFlag
repair = NoRepair,
817 CheckSigsFlag checkSigs = CheckSigs,
818 SubstituteFlag substitute = NoSubstitute);
820std::map<StorePath, StorePath> copyPaths(
822 const StorePathSet & paths,
823 RepairFlag
repair = NoRepair,
824 CheckSigsFlag checkSigs = CheckSigs,
825 SubstituteFlag substitute = NoSubstitute);
832 const std::set<RealisedPath> & paths,
833 RepairFlag
repair = NoRepair,
834 CheckSigsFlag checkSigs = CheckSigs,
835 SubstituteFlag substitute = NoSubstitute);
839 const StorePathSet & paths,
840 RepairFlag
repair = NoRepair,
841 CheckSigsFlag checkSigs = CheckSigs,
842 SubstituteFlag substitute = NoSubstitute);
857OutputPathMap resolveDerivedPath(
Store &,
const DerivedPath::Built &,
Store * evalStore =
nullptr);
871ref<Store> openStore(
const std::string & uri = settings.storeUri.get(),
872 const Store::Params & extraParams = Store::Params());
879std::list<ref<Store>> getDefaultSubstituters();
883 std::set<std::string> uriSchemes;
888 std::function<std::shared_ptr<Store> (
889 std::string_view scheme,
890 std::string_view authorityPath,
892 std::function<std::shared_ptr<StoreConfig> ()> getConfig;
897 static std::vector<StoreFactory> * registered;
899 template<
typename T,
typename TConfig>
902 if (!registered) registered =
new std::vector<StoreFactory>();
904 .uriSchemes = TConfig::uriSchemes(),
906 ([](
auto scheme,
auto uri,
auto & params)
907 -> std::shared_ptr<Store>
908 {
return std::make_shared<T>(scheme, uri, params); }),
911 -> std::shared_ptr<StoreConfig>
912 {
return std::make_shared<TConfig>(StringMap({})); })
914 registered->push_back(factory);
918template<
typename T,
typename TConfig>
919struct RegisterStoreImplementation
921 RegisterStoreImplementation()
923 Implementations::add<T, TConfig>();
932std::string showPaths(
const PathSet & paths);
935std::optional<ValidPathInfo> decodeValidPathInfo(
938 std::optional<HashResult> hashGiven = std::nullopt);
942std::map<DrvOutput, StorePath> drvOutputReferences(
946 Store * evalStore =
nullptr);
Definition callback.hh:17
Definition lru-cache.hh:16
Definition nar-info-disk-cache.hh:11
Definition store-api.hh:169
std::optional< StorePath > getBuildDerivationPath(const StorePath &)
Definition store-api.cc:1243
virtual std::string getUri()=0
Definition store-api.cc:444
virtual std::map< std::string, std::optional< StorePath > > queryStaticPartialDerivationOutputMap(const StorePath &path)
Definition store-api.cc:458
ref< const ValidPathInfo > queryPathInfo(const StorePath &path)
Definition store-api.cc:613
virtual StorePathSet queryValidDerivers(const StorePath &path)
Definition store-api.hh:353
virtual void addSignatures(const StorePath &storePath, const StringSet &sigs)
Definition store-api.hh:622
virtual unsigned int getProtocol()
Definition store-api.hh:741
Derivation readInvalidDerivation(const StorePath &drvPath)
Definition store-api.cc:1274
virtual void queryReferrers(const StorePath &path, StorePathSet &referrers)
Definition store-api.hh:342
std::optional< std::shared_ptr< const ValidPathInfo > > queryPathInfoFromClientCache(const StorePath &path)
Definition store-api.cc:638
virtual void ensurePath(const StorePath &path)
Definition entry-points.cc:106
virtual void computeFSClosure(const StorePathSet &paths, StorePathSet &out, bool flipDirection=false, bool includeOutputs=false, bool includeDerivers=false)
Definition misc.cc:17
virtual void buildPaths(const std::vector< DerivedPath > &paths, BuildMode buildMode=bmNormal, std::shared_ptr< Store > evalStore=nullptr)
Definition entry-points.cc:11
virtual void querySubstitutablePathInfos(const StorePathCAMap &paths, SubstitutablePathInfos &infos)
Definition store-api.cc:518
std::vector< std::pair< ValidPathInfo, std::unique_ptr< Source > > > PathsSource
Definition store-api.hh:417
virtual void connect()
Definition store-api.hh:736
virtual void registerDrvOutput(const Realisation &output)
Definition store-api.hh:498
StorePath followLinksToStorePath(std::string_view path) const
Definition store-api.cc:64
StorePaths topoSortPaths(const StorePathSet &paths)
Definition misc.cc:300
virtual void addTempRoot(const StorePath &path)
Definition store-api.hh:583
virtual BuildResult buildDerivation(const StorePath &drvPath, const BasicDerivation &drv, BuildMode buildMode=bmNormal)
Definition entry-points.cc:80
OutputPathMap queryDerivationOutputMap(const StorePath &path, Store *evalStore=nullptr)
Definition store-api.cc:496
virtual std::map< std::string, std::optional< StorePath > > queryPartialDerivationOutputMap(const StorePath &path, Store *evalStore=nullptr)
Definition store-api.cc:468
bool isValidPath(const StorePath &path)
Definition store-api.cc:568
virtual StorePathSet queryValidPaths(const StorePathSet &paths, SubstituteFlag maybeSubstitute=NoSubstitute)
Definition store-api.cc:807
virtual void optimiseStore()
Definition store-api.hh:598
Derivation derivationFromPath(const StorePath &drvPath)
Definition store-api.cc:1225
Path followLinksToStore(std::string_view path) const
Definition store-api.cc:50
virtual bool verifyStore(bool checkContents, RepairFlag repair=NoRepair)
Definition store-api.hh:605
virtual void setOptions()
Definition store-api.hh:771
virtual std::optional< StorePath > queryPathFromHashPart(const std::string &hashPart)=0
StorePaths importPaths(Source &source, CheckSigsFlag checkSigs=CheckSigs)
Definition export-import.cc:52
std::shared_ptr< const Realisation > queryRealisation(const DrvOutput &)
Definition store-api.cc:768
virtual StorePathSet querySubstitutablePaths(const StorePathSet &paths)
Definition store-api.hh:395
virtual std::vector< KeyedBuildResult > buildPathsWithResults(const std::vector< DerivedPath > &paths, BuildMode buildMode=bmNormal, std::shared_ptr< Store > evalStore=nullptr)
Definition entry-points.cc:50
virtual StorePathSet queryAllValidPaths()
Definition store-api.hh:269
virtual void queryMissing(const std::vector< DerivedPath > &targets, StorePathSet &willBuild, StorePathSet &willSubstitute, StorePathSet &unknown, uint64_t &downloadSize, uint64_t &narSize)
Definition misc.cc:100
virtual std::optional< TrustedFlag > isTrustedClient()=0
virtual void addToStore(const ValidPathInfo &info, Source &narSource, RepairFlag repair=NoRepair, CheckSigsFlag checkSigs=CheckSigs)=0
virtual StorePathSet queryDerivationOutputs(const StorePath &path)
Definition store-api.cc:507
virtual void repairPath(const StorePath &path)
Definition entry-points.cc:127
virtual ref< SourceAccessor > getFSAccessor(bool requireValidPath=true)=0
Derivation readDerivation(const StorePath &drvPath)
Definition store-api.cc:1271
virtual void addMultipleToStore(Source &source, RepairFlag repair=NoRepair, CheckSigsFlag checkSigs=CheckSigs)
Definition store-api.cc:303
void substitutePaths(const StorePathSet &paths)
Definition store-api.cc:785
StorePathSet exportReferences(const StorePathSet &storePaths, const StorePathSet &inputPaths)
Definition store-api.cc:896
void clearPathInfoCache()
Definition store-api.hh:727
virtual StorePath addToStoreFromDump(Source &dump, std::string_view name, FileSerialisationMethod dumpMethod=FileSerialisationMethod::NixArchive, ContentAddressMethod hashMethod=ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo=HashAlgorithm::SHA256, const StorePathSet &references=StorePathSet(), RepairFlag repair=NoRepair)=0
virtual void narFromPath(const StorePath &path, Sink &sink)=0
ValidPathInfo addToStoreSlow(std::string_view name, const SourcePath &path, ContentAddressMethod method=ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo=HashAlgorithm::SHA256, const StorePathSet &references=StorePathSet(), std::optional< Hash > expectedCAHash={})
Definition store-api.cc:346
void unsupported(const std::string &op)
Definition store-api.hh:786
void exportPaths(const StorePathSet &paths, Sink &sink)
Definition export-import.cc:11
virtual bool pathInfoIsUntrusted(const ValidPathInfo &)
Definition store-api.hh:319
std::string makeValidityRegistration(const StorePathSet &paths, bool showDerivers, bool showHash)
Definition store-api.cc:868
virtual void init()
Definition store-api.hh:215
FileSerialisationMethod
Definition file-content-address.hh:20
@ NixArchive
Definition file-content-address.hh:36
std::function< bool(const Path &path)> PathFilter
Definition file-system.hh:365
ChunkedVector< std::string, 8192 > store
Definition lexer.l:1011
Path toRealPath(const Path &path, const NixStringContext &context)
const Activity & act
Definition lexer.l:2371
std::ostream & str
Definition lexer.l:1728
RepairFlag repair
Definition lexer.l:7173
std::string path
Definition lexer.l:1399
const uint32_t exportMagic
Definition store-api.hh:88
Definition logging.hh:137
Definition derivations.hh:285
Definition build-result.hh:14
Definition content-address.hh:31
Definition content-address.hh:153
Definition derivations.hh:342
Definition realisation.hh:24
Definition store-api.hh:896
Definition build-result.hh:121
Definition realisation.hh:49
Definition realisation.hh:118
Definition derived-path.hh:102
Definition serialise.hh:20
Definition source-accessor.hh:42
Definition source-path.hh:22
Definition serialise.hh:68
virtual const std::string name()=0
virtual std::string doc()
Definition store-api.hh:120
virtual std::optional< ExperimentalFeature > experimentalFeature() const
Definition store-api.hh:129
Definition store-api.hh:882
std::function< std::shared_ptr< Store >(std::string_view scheme, std::string_view authorityPath, const Store::Params ¶ms)> create
Definition store-api.hh:891
Definition store-reference.hh:43
Definition store-api.hh:172
std::shared_ptr< const ValidPathInfo > value
Definition store-api.hh:182
bool didExist()
Definition store-api.hh:194
bool isKnownNow()
Definition store-api.cc:449
std::chrono::time_point< std::chrono::steady_clock > time_point
Definition store-api.hh:177
Definition store-api.hh:200
Definition store-api.hh:692
Definition path-info.hh:130
std::string Path
Definition types.hh:22