rebar3/src/rebar_pkg_resource.erl

300 lines
13 KiB
Erlang

%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 et
-module(rebar_pkg_resource).
-behaviour(rebar_resource_v2).
-export([init/2,
lock/2,
download/4,
download/5,
needs_update/2,
make_vsn/2,
format_error/1]).
-ifdef(TEST).
%% exported for test purposes
-export([store_etag_in_cache/2]).
-endif.
-include("rebar.hrl").
-include_lib("providers/include/providers.hrl").
-type package() :: {pkg, binary(), binary(), binary(), binary(), rebar_hex_repos:repo()}.
%%==============================================================================
%% Public API
%%==============================================================================
-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}.
init(Type, State) ->
{ok, Vsn} = application:get_key(rebar, vsn),
BaseConfig = #{http_adapter => {rebar_httpc_adapter, #{profile => rebar}},
http_user_agent_fragment =>
<<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>},
Repos = rebar_hex_repos:from_state(BaseConfig, State),
Resource = rebar_resource_v2:new(Type, ?MODULE, #{repos => Repos,
base_config => BaseConfig}),
{ok, Resource}.
-spec lock(AppInfo, ResourceState) -> Res when
AppInfo :: rebar_app_info:t(),
ResourceState :: rebar_resource_v2:resource_state(),
Res :: {atom(), string(), any(), binary(), binary()}.
lock(AppInfo, _) ->
{pkg, Name, Vsn, OldHash, Hash, _RepoConfig} = rebar_app_info:source(AppInfo),
{pkg, Name, Vsn, OldHash, Hash}.
%%------------------------------------------------------------------------------
%% @doc
%% Return true if the stored version of the pkg is older than the current
%% version.
%% @end
%%------------------------------------------------------------------------------
-spec needs_update(AppInfo, ResourceState) -> Res when
AppInfo :: rebar_app_info:t(),
ResourceState :: rebar_resource_v2:resource_state(),
Res :: boolean().
needs_update(AppInfo, _) ->
{pkg, _Name, Vsn, _OldHash, _Hash, _} = rebar_app_info:source(AppInfo),
rebar_utils:to_binary(rebar_app_info:original_vsn(AppInfo)) =/= rebar_utils:to_binary(Vsn).
%%------------------------------------------------------------------------------
%% @doc
%% Download the given pkg.
%% @end
%%------------------------------------------------------------------------------
-spec download(TmpDir, AppInfo, State, ResourceState) -> Res when
TmpDir :: file:name(),
AppInfo :: rebar_app_info:t(),
ResourceState :: rebar_resource_v2:resource_state(),
State :: rebar_state:t(),
Res :: ok | {error,_}.
download(TmpDir, AppInfo, State, ResourceState) ->
case download(TmpDir, rebar_app_info:source(AppInfo), State, ResourceState, true) of
ok ->
ok;
Error ->
{error, Error}
end.
%%------------------------------------------------------------------------------
%% @doc
%% Download the given pkg. The etag belonging to the pkg file will be updated
%% only if the UpdateEtag is true and the ETag returned from the hexpm server
%% is different.
%% @end
%%------------------------------------------------------------------------------
-spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when
TmpDir :: file:name(),
Pkg :: package(),
State :: rebar_state:t(),
ResourceState:: rebar_resource_v2:resource_state(),
UpdateETag :: boolean(),
Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} |
{fetch_fail, binary(), binary()}.
download(TmpDir, Pkg={pkg, Name, Vsn, _OldHash, _Hash, Repo}, State, _ResourceState, UpdateETag) ->
{ok, PackageDir} = rebar_packages:package_dir(Repo, State),
Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
ETagFile = binary_to_list(<<Name/binary, "-", Vsn/binary, ".etag">>),
CachePath = filename:join(PackageDir, Package),
ETagPath = filename:join(PackageDir, ETagFile),
case cached_download(TmpDir, CachePath, Pkg, State, etag(CachePath, ETagPath), ETagPath, UpdateETag) of
{bad_registry_checksum, Expected, Found} ->
%% checksum comparison failed. in case this is from a modified cached package
%% overwrite the etag if it exists so it is not relied on again
store_etag_in_cache(ETagPath, <<>>),
?PRV_ERROR({bad_registry_checksum, Name, Vsn, Expected, Found});
Result ->
Result
end.
%%------------------------------------------------------------------------------
%% @doc
%% Implementation of rebar_resource make_vsn callback.
%% Returns {error, string()} as this operation is not supported for pkg sources.
%% @end
%%------------------------------------------------------------------------------
-spec make_vsn(AppInfo, ResourceState) -> Res when
AppInfo :: rebar_app_info:t(),
ResourceState :: rebar_resource_v2:resource_state(),
Res :: {'error', string()}.
make_vsn(_, _) ->
{error, "Replacing version of type pkg not supported."}.
format_error({bad_registry_checksum, Name, Vsn, Expected, Found}) ->
io_lib:format("The checksum for package at ~ts-~ts (~ts) does not match the "
"checksum expected from the registry (~ts). "
"Run `rebar3 do unlock ~ts, update` and then try again.",
[Name, Vsn, Found, Expected, Name]).
%%------------------------------------------------------------------------------
%% @doc
%% Download the pkg belonging to the given address. If the etag of the pkg
%% is the same what we stored in the etag file previously return {ok, cached},
%% if the file has changed (so the etag is not the same anymore) return
%% {ok, Contents, NewEtag}, otherwise if some error occurred return error.
%% @end
%%------------------------------------------------------------------------------
-spec request(rebar_hex_repos:repo(), binary(), binary(), false | binary())
-> {ok, cached} | {ok, binary(), binary()} | error.
request(Config, Name, Version, ETag) ->
Config1 = Config#{http_etag => ETag},
try r3_hex_repo:get_tarball(Config1, Name, Version) of
{ok, {200, #{<<"etag">> := ETag1}, Tarball}} ->
{ok, Tarball, ETag1};
{ok, {304, _Headers, _}} ->
{ok, cached};
{ok, {Code, _Headers, _Body}} ->
?DEBUG("Request for package ~s-~s failed: status code ~p", [Name, Version, Code]),
error;
{error, Reason} ->
?DEBUG("Request for package ~s-~s failed: ~p", [Name, Version, Reason]),
error
catch
_:Exception ->
?DEBUG("hex_repo:get_tarball failed: ~p", [Exception]),
error
end.
%%------------------------------------------------------------------------------
%% @doc
%% Read the etag belonging to the pkg file from the cache directory. The etag
%% is stored in a separate file when the etag belonging to the package is
%% returned from the hexpm server. The name is package-vsn.etag.
%% @end
%%------------------------------------------------------------------------------
-spec etag(PackagePath, ETagPath) -> Res when
PackagePath :: file:name(),
ETagPath :: file:name(),
Res :: binary().
etag(PackagePath, ETagPath) ->
case file:read_file(ETagPath) of
{ok, Bin} ->
%% just in case a user deleted a cached package but not its etag
%% verify the package is also there, and if not, ignore the etag
case filelib:is_file(PackagePath) of
true ->
Bin;
false ->
<<>>
end;
{error, _} ->
<<>>
end.
%%------------------------------------------------------------------------------
%% @doc
%% Store the given etag in the .cache folder. The name is pakckage-vsn.etag.
%% @end
%%------------------------------------------------------------------------------
-spec store_etag_in_cache(File, ETag) -> Res when
File :: file:name(),
ETag :: binary(),
Res :: ok.
store_etag_in_cache(Path, ETag) ->
_ = file:write_file(Path, ETag).
%%%=============================================================================
%%% Private functions
%%%=============================================================================
-spec cached_download(TmpDir, CachePath, Pkg, State, ETag, ETagPath, UpdateETag) -> Res when
TmpDir :: file:name(),
CachePath :: file:name(),
Pkg :: package(),
State :: rebar_state:t(),
ETag :: binary(),
ETagPath :: file:name(),
UpdateETag :: boolean(),
Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}
| {bad_registry_checksum, integer(), integer()} | {error, _}.
cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _OldHash, _Hash, RepoConfig}, _State, ETag,
ETagPath, UpdateETag) ->
?DEBUG("Making request to get package ~ts from repo ~ts",
[Name, rebar_hex_repos:format_repo(RepoConfig)]),
case request(RepoConfig, Name, Vsn, ETag) of
{ok, cached} ->
?DEBUG("Version cached at ~ts is up to date, reusing it", [CachePath]),
serve_from_cache(TmpDir, CachePath, Pkg);
{ok, Body, NewETag} ->
?DEBUG("Downloaded package ~ts, caching at ~ts", [Name, CachePath]),
maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag),
serve_from_download(TmpDir, CachePath, Pkg, Body);
error when ETag =/= <<>> ->
store_etag_in_cache(ETagPath, ETag),
?INFO("Download error, using cached file at ~ts", [CachePath]),
serve_from_cache(TmpDir, CachePath, Pkg);
error ->
{fetch_fail, Name, Vsn}
end.
-spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when
TmpDir :: file:name(),
CachePath :: file:name(),
Pkg :: package(),
Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}.
serve_from_cache(TmpDir, CachePath, Pkg) ->
{ok, Binary} = file:read_file(CachePath),
serve_from_memory(TmpDir, Binary, Pkg).
-spec serve_from_memory(TmpDir, Tarball, Package) -> Res when
TmpDir :: file:name(),
Tarball :: binary(),
Package :: package(),
Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}.
serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, OldHash, Hash, _RepoConfig}) ->
RegistryChecksum = list_to_integer(binary_to_list(Hash), 16),
OldRegistryChecksum = maybe_old_registry_checksum(OldHash),
case r3_hex_tarball:unpack(Binary, TmpDir) of
{ok, #{outer_checksum := <<Checksum:256/big-unsigned>>} = Res} when RegistryChecksum =/= Checksum ->
#{inner_checksum := <<OldChecksum:256/big-unsigned>>} = Res,
%% Not triggerable in tests, but code feels logically wrong without it since inner checksums are not hard
%% deprecated. This logic should be removed when inner checksums do become hard deprecated and/or no longer
%% supported by rebar3.
case OldRegistryChecksum == OldChecksum of
true ->
?DEBUG("Expected hash ~64.16.0B does not match outer checksum of fetched package ~64.16.0B, but
matches inner checksum ~64.16.0B",
[RegistryChecksum, Checksum, OldChecksum]),
{bad_registry_checksum, RegistryChecksum, OldChecksum};
false ->
?DEBUG("Expected hash ~64.16.0B does not match outer checksum or inner checksum of fetched package
~64.16.0B / ~64.16.0B", [RegistryChecksum, Checksum, OldChecksum]),
{bad_registry_checksum, RegistryChecksum, Checksum}
end;
{ok, #{outer_checksum := <<RegistryChecksum:256/big-unsigned>>}} ->
ok;
{error, Reason} ->
{error, {hex_tarball, Reason}}
end.
maybe_old_registry_checksum(undefined) -> undefined;
maybe_old_registry_checksum(Hash) -> list_to_integer(binary_to_list(Hash), 16).
-spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when
TmpDir :: file:name(),
CachePath :: file:name(),
Package :: package(),
Binary :: binary(),
Res :: ok | {error,_}.
serve_from_download(TmpDir, CachePath, Package, Binary) ->
?DEBUG("Writing ~p to cache at ~ts", [catch anon(Package), CachePath]),
file:write_file(CachePath, Binary),
serve_from_memory(TmpDir, Binary, Package).
-spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when
UpdateETag :: boolean(),
Path :: file:name(),
ETag :: binary(),
Res :: ok.
maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) ->
ok;
maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) ->
store_etag_in_cache(Path, ETag).
anon({pkg, PkgName, PkgVsn, OldHash, Hash, RepoConfig}) ->
{pkg, PkgName, PkgVsn, OldHash, Hash,
rebar_hex_repos:anon_repo_config(RepoConfig)}.