%%% @doc utility functions to do the basic discovery of apps %%% and layout for the project. -module(rebar_app_discover). -export([do/2, format_error/1, find_unbuilt_apps/2, find_apps/2, find_apps/3, find_apps/4, find_app/3, find_app/4]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -type app_resource_type() :: app | app_src | script | mix_exs. %% @doc from the base directory, find all the applications %% at the top level and their dependencies based on the configuration %% and profile information. -spec do(rebar_state:t(), [file:filename()]) -> rebar_state:t(). do(State, LibDirs) -> BaseDir = rebar_state:dir(State), Dirs = [filename:join(BaseDir, LibDir) || LibDir <- LibDirs], RebarOpts = rebar_state:opts(State), SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]), Apps = find_apps(Dirs, SrcDirs, all, State), ProjectDeps = rebar_state:deps_names(State), DepsDir = rebar_dir:deps_dir(State), CurrentProfiles = rebar_state:current_profiles(State), %% There may be a top level src which is an app and there may not %% Find it here if there is, otherwise define the deps parent as root TopLevelApp = define_root_app(Apps, State), ?DEBUG("Found top-level apps: [~ts]~n\tusing config: ~p", [case lists:flatten([[",",rebar_app_info:name(App)] || App <- Apps]) of [] -> ""; Str -> tl(Str) end, [{src_dirs, SrcDirs}, {lib_dirs, LibDirs}]]), %% Handle top level deps State1 = lists:foldl(fun(Profile, StateAcc) -> ProfileDeps = rebar_state:get(StateAcc, {deps, Profile}, []), ProfileDeps2 = rebar_utils:tup_dedup(ProfileDeps), StateAcc1 = rebar_state:set(StateAcc, {deps, Profile}, ProfileDeps2), ParsedDeps = parse_profile_deps(Profile ,TopLevelApp ,ProfileDeps2 ,rebar_state:opts(StateAcc1) ,StateAcc1), rebar_state:set(StateAcc1, {parsed_deps, Profile}, ParsedDeps) end, State, lists:reverse(CurrentProfiles)), %% Handle sub project apps deps %% Sort apps so we get the same merged deps config every time SortedApps = rebar_utils:sort_deps(Apps), lists:foldl(fun(AppInfo, StateAcc) -> Name = rebar_app_info:name(AppInfo), case enable(State, AppInfo) of true -> {AppInfo1, StateAcc1} = merge_opts(TopLevelApp, AppInfo, StateAcc), OutDir = filename:join(DepsDir, Name), AppInfo2 = rebar_app_info:out_dir(AppInfo1, OutDir), ProjectDeps1 = lists:delete(Name, ProjectDeps), rebar_state:project_apps(StateAcc1 ,rebar_app_info:deps(AppInfo2, ProjectDeps1)); false -> ?INFO("Ignoring ~ts", [Name]), StateAcc end end, State1, SortedApps). %% @doc checks whether there is an app at the top level (and returns its %% name) or the 'root' atom in case we're in an umbrella project. -spec define_root_app([rebar_app_info:t()], rebar_state:t()) -> root | binary(). define_root_app(Apps, State) -> RootDir = rebar_dir:root_dir(State), case ec_lists:find(fun(X) -> ec_file:real_dir_path(rebar_app_info:dir(X)) =:= ec_file:real_dir_path(RootDir) end, Apps) of {ok, App} -> rebar_app_info:name(App); error -> root end. %% @doc formatting errors from the module. -spec format_error(term()) -> iodata(). format_error({module_list, File}) -> io_lib:format("Error reading module list from ~p~n", [File]); format_error({missing_module, Module}) -> io_lib:format("Module defined in app file missing: ~p~n", [Module]); format_error({cannot_read_app_file, AppFile}) -> io_lib:format("Cannot read app file: ~p~n", [AppFile]); format_error({bad_term_file, _File, _Reason} = Error) -> rebar_file_utils:format_error(Error). %% @doc merges configuration of a project app and the top level state %% some configuration like erl_opts must be merged into a subapp's opts %% while plugins and hooks need to be kept defined to only either the %% top level state or an individual application. -spec merge_opts(root | binary(), rebar_app_info:t(), rebar_state:t()) -> {rebar_app_info:t(), rebar_state:t()}. merge_opts(TopLevelApp, AppInfo, State) -> %% These steps make sure that hooks and artifacts are run in the context of %% the application they are defined at. If an umbrella structure is used and %% they are defined at the top level they will instead run in the context of %% the State and at the top level, not as part of an application. CurrentProfiles = rebar_state:current_profiles(State), {AppInfo1, State1} = maybe_reset_hooks_plugins(AppInfo, State), Name = rebar_app_info:name(AppInfo1), %% We reset the opts here to default so no profiles are applied multiple times AppInfo2 = case TopLevelApp of Name -> % don't apply to the root app AppInfo; _ -> % apply overrides when in an umbrella project or on deps rebar_app_info:apply_overrides(rebar_state:get(State1, overrides, []), AppInfo1) end, AppInfo3 = rebar_app_info:apply_profiles(AppInfo2, CurrentProfiles), %% Will throw an exception if checks fail rebar_app_info:verify_otp_vsn(AppInfo3), State2 = lists:foldl(fun(Profile, StateAcc) -> handle_profile(Profile, Name, AppInfo3, StateAcc) end, State1, lists:reverse(CurrentProfiles)), {AppInfo3, State2}. %% @doc Applies a given profile for an app, ensuring the deps %% match the context it will require. -spec handle_profile(atom(), binary(), rebar_app_info:t(), rebar_state:t()) -> rebar_state:t(). handle_profile(Profile, Name, AppInfo, State) -> TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []), AppProfileDeps = rebar_app_info:get(AppInfo, {deps, Profile}, []), AppProfileDeps2 = rebar_utils:tup_dedup(AppProfileDeps), ProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_umerge(TopLevelProfileDeps ,AppProfileDeps2)), State1 = rebar_state:set(State, {deps, Profile}, ProfileDeps2), %% Only deps not also specified in the top level config need %% to be included in the parsed deps NewDeps = ProfileDeps2 -- TopLevelProfileDeps, ParsedDeps = parse_profile_deps(Profile, Name, NewDeps, rebar_app_info:opts(AppInfo), State1), State2 = rebar_state:set(State1, {deps, Profile}, ProfileDeps2), rebar_state:set(State2, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps). %% @doc parses all the known dependencies for a given profile -spec parse_profile_deps(Profile, Name, Deps, Opts, rebar_state:t()) -> [rebar_app_info:t()] when Profile :: atom(), Name :: binary(), Deps :: [term()], % TODO: refine types Opts :: term(). % TODO: refine types parse_profile_deps(Profile, Name, Deps, Opts, State) -> DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile), Locks = rebar_state:get(State, {locks, Profile}, []), rebar_app_utils:parse_deps(Name ,DepsDir ,Deps ,rebar_state:opts(State, Opts) ,Locks ,1). %% reset the State hooks if there is a top level application -spec maybe_reset_hooks_plugins(AppInfo, State) -> {AppInfo, State} when AppInfo :: rebar_app_info:t(), State :: rebar_state:t(). maybe_reset_hooks_plugins(AppInfo, State) -> Dir = rebar_app_info:dir(AppInfo), CurrentProfiles = rebar_state:current_profiles(State), case ec_file:real_dir_path(rebar_dir:root_dir(State)) of Dir -> Opts = reset_hooks(rebar_state:opts(State), CurrentProfiles), State1 = rebar_state:opts(State, Opts), %% set plugins to empty since this is an app at the top level %% and top level plugins are installed in run_aux AppInfo1 = rebar_app_info:set(rebar_app_info:set(AppInfo, {plugins,default}, []), plugins, []), {AppInfo1, State1}; _ -> %% if not in the top root directory then we need to merge in the %% default state opts to this subapp's opts Default = reset_hooks(rebar_state:default(State), CurrentProfiles), AppInfo1 = rebar_app_info:update_opts(AppInfo, Default), {AppInfo1, State} end. %% @doc make the hooks empty for a given set of options -spec reset_hooks(Opts, Profiles) -> Opts when Opts :: rebar_dict(), Profiles :: [atom()]. reset_hooks(Opts, CurrentProfiles) -> AllHooks = [post_hooks, pre_hooks, provider_hooks, artifacts], Opts1 = lists:foldl(fun(Key, OptsAcc) -> rebar_opts:set(OptsAcc, Key, []) end, Opts, AllHooks), Profiles = rebar_opts:get(Opts1, profiles, []), Profiles1 = lists:map(fun({P, ProfileOpts}) -> case lists:member(P, CurrentProfiles) of true -> {P, [X || X={Key, _} <- ProfileOpts, not lists:member(Key, AllHooks)]}; false -> {P, ProfileOpts} end end, Profiles), rebar_opts:set(Opts1, profiles, Profiles1). %% @private find the directories for all apps, while detecting their source dirs %% Returns the app dir with the respective src_dirs for them, in that order, %% for every app found. -spec all_app_dirs([file:name()], rebar_state:t()) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs, State) -> lists:flatmap(fun(LibDir) -> case filelib:is_dir(LibDir) of true -> {_, SrcDirs} = find_config_src(LibDir, ["src"]), app_dirs(LibDir, SrcDirs, State); false -> [] end end, LibDirs). %% @private find the directories for all apps based on their source dirs %% Returns the app dir with the respective src_dirs for them, in that order, %% for every app found. -spec all_app_dirs([file:name()], [file:name()], rebar_state:t()) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs, SrcDirs, State) -> lists:flatmap(fun(LibDir) -> app_dirs(LibDir, SrcDirs, State) end, LibDirs). %% @private find the directories based on the library directories. %% Returns the app dir with the respective src_dirs for them, in that order, %% for every app found. %% %% The function returns the src directories since they might have been %% detected in a top-level loop and we want to skip further detection %% starting now. -spec app_dirs([file:name()], [file:name()], rebar_state:t()) -> [{file:name(), [file:name()]}]. app_dirs(LibDir, SrcDirs, State) -> Extensions = rebar_state:get(State, application_resource_extensions, ?DEFAULT_APP_RESOURCE_EXT), Paths = lists:append([ [filename:join([LibDir, SrcDir, "*" ++ Ext]) || Ext <- Extensions ] || SrcDir <- SrcDirs ]), EbinPath = filename:join([LibDir, "ebin", "*.app"]), MixExsPath = filename:join([LibDir, "mix.exs"]), lists:usort(lists:foldl(fun(Path, Acc) -> Files = filelib:wildcard(rebar_utils:to_list(Path)), [{app_dir(File), SrcDirs} || File <- Files] ++ Acc end, [], [EbinPath, MixExsPath | Paths])). %% @doc find all apps that haven't been built in a list of directories -spec find_unbuilt_apps([file:filename_all()], rebar_state:t()) -> [rebar_app_info:t()]. find_unbuilt_apps(LibDirs, State) -> find_apps(LibDirs, invalid, State). %% @doc for each directory passed, find all apps that are valid. %% Returns all the related app info records. -spec find_apps([file:filename_all()], rebar_state:t()) -> [rebar_app_info:t()]. find_apps(LibDirs, State) -> find_apps(LibDirs, valid, State). %% @doc for each directory passed, find all apps according %% to the validity rule passed in. Returns all the related %% app info records. -spec find_apps([file:filename_all()], valid | invalid | all, rebar_state:t()) -> [rebar_app_info:t()]. find_apps(LibDirs, Validate, State) -> rebar_utils:filtermap( fun({AppDir, AppSrcDirs}) -> find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate, State) end, all_app_dirs(LibDirs, State) ). %% @doc for each directory passed, with the configured source directories, %% find all apps according to the validity rule passed in. %% Returns all the related app info records. -spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all, rebar_state:t()) -> [rebar_app_info:t()]. find_apps(LibDirs, SrcDirs, Validate, State) -> rebar_utils:filtermap( fun({AppDir, AppSrcDirs}) -> find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate, State) end, all_app_dirs(LibDirs, SrcDirs, State) ). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. Returns the related %% app info record. -spec find_app(file:filename_all(), valid | invalid | all, rebar_state:t()) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate, State) -> {Config, SrcDirs} = find_config_src(AppDir, ["src"]), AppInfo = rebar_app_info:update_opts(rebar_app_info:dir(rebar_app_info:new(), AppDir), dict:new(), Config), find_app_(AppInfo, AppDir, SrcDirs, Validate, State). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. Returns the related %% app info record. -spec find_app(rebar_app_info:t(), file:filename_all(), valid | invalid | all, rebar_state:t()) -> {true, rebar_app_info:t()} | false. find_app(AppInfo, AppDir, Validate, State) -> %% if no src dir is passed, figure it out from the app info, with a default %% of src/ AppOpts = rebar_app_info:opts(AppInfo), SrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]), find_app_(AppInfo, AppDir, SrcDirs, Validate, State). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. The third argument includes %% the directories where source files can be located. Returns the related %% app info record. -spec find_app(rebar_app_info:t(), file:filename_all(), [file:filename_all()], valid | invalid | all, rebar_state:t()) -> {true, rebar_app_info:t()} | false. find_app(AppInfo, AppDir, SrcDirs, Validate, State) -> AppInfo1 = case ec_file:real_dir_path(rebar_dir:root_dir(State)) of AppDir -> Opts = rebar_state:opts(State), rebar_app_info:default(rebar_app_info:opts(AppInfo, Opts), Opts); _ -> Config = rebar_config:consult(AppDir), rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config) end, find_app_(AppInfo1, AppDir, SrcDirs, Validate, State). -spec find_app_(rebar_app_info:t(), file:filename_all(), [file:filename_all()], valid | invalid | all, rebar_state:t()) -> {true, rebar_app_info:t()} | false. find_app_(AppInfo, AppDir, SrcDirs, Validate, State) -> Extensions = rebar_state:get(State, application_resource_extensions, ?DEFAULT_APP_RESOURCE_EXT), NormSrcDirs = [case SrcDir of {ActualSrcDir, _Opts} -> ActualSrcDir; _ -> SrcDir end || SrcDir <- SrcDirs], ResourceFiles = [ {app, filelib:wildcard(filename:join([AppDir, "ebin", "*.app"]))}, {mix_exs, filelib:wildcard(filename:join([AppDir, "mix.exs"]))} | [{extension_type(Ext), lists:append([filelib:wildcard(filename:join([AppDir, SrcDir, "*" ++ Ext])) || SrcDir <- NormSrcDirs])} || Ext <- Extensions] ], FlattenedResourceFiles = flatten_resource_files(ResourceFiles), try_handle_resource_files(AppInfo, AppDir, FlattenedResourceFiles, Validate). -spec extension_type(string()) -> app_resource_type(). extension_type(Extension) -> Mapping = [ {".app", app}, {".src", app_src}, {".script", script}, {".exs", mix_exs} ], extension_type(Mapping, Extension). -spec extension_type([{string(), Type}], string()) -> Type when Type :: app_resource_type(). extension_type([], _) -> %% default to app_src app_src; extension_type([{Pattern, Type} | Rest], Extension) -> case lists:suffix(Pattern, Extension) of true -> Type; false -> extension_type(Rest, Extension) end. -spec flatten_resource_files(ResourceFiles) -> FlattenedResourceFiles when ResourceFiles :: [{app_resource_type(), [file:filename()]}], FlattenedResourceFiles :: [{app_resource_type(), file:filename()}]. flatten_resource_files(ResourceFiles) -> {Flattened, _} = lists:foldl( fun flatten_resource_impl/2, {[], []}, ResourceFiles), lists:reverse(Flattened). flatten_resource_impl({Type, Files}, Acc = {ResAcc, Used}) -> NewFiles = [F || F <- Files, not lists:member(F, Used)], case NewFiles of [] -> Acc; [File] -> {[{Type, File} | ResAcc], [File | Used]}; Others -> throw({error, {multiple_app_files, Others}}) end. %% @doc find the directory that an appfile has -spec app_dir(file:filename()) -> file:filename(). app_dir(AppFile) -> filename:join(rebar_utils:droplast(filename:split(filename:dirname(AppFile)))). %% @doc populates an app info record based on an app directory and its %% app file. -spec create_app_info(rebar_app_info:t(), file:name(), file:name()) -> rebar_app_info:t(). create_app_info(AppInfo, AppDir, AppFile) -> try rebar_config:consult_app_file(AppFile) of [{application, AppName, AppDetails}] -> AppVsn = proplists:get_value(vsn, AppDetails), Applications = proplists:get_value(applications, AppDetails, []), IncludedApplications = proplists:get_value(included_applications, AppDetails, []), OptionalApplications = proplists:get_value(optional_applications, AppDetails, []), AppInfo1 = rebar_app_info:name( rebar_app_info:vsn( rebar_app_info:original_vsn( rebar_app_info:dir(AppInfo, AppDir), AppVsn), AppVsn), AppName), AppInfo2 = rebar_app_info:applications( rebar_app_info:app_details(AppInfo1, AppDetails), Applications), AppInfo3 = rebar_app_info:included_applications(AppInfo2, IncludedApplications), AppInfo4 = rebar_app_info:optional_applications(AppInfo3, OptionalApplications), Valid = case rebar_app_utils:validate_application_info(AppInfo4) =:= true andalso rebar_app_info:has_all_artifacts(AppInfo4) =:= true of true -> true; _ -> false end, rebar_app_info:dir(rebar_app_info:valid(AppInfo4, Valid), AppDir); _Invalid -> throw({error, {?MODULE, {cannot_read_app_file, AppFile}}}) catch throw:{error, {rebar_file_utils, Err = {bad_term_file, _File, _Reason}}} -> throw({error, {?MODULE, Err}}) % wrap this end. %% @doc Read in and parse the .app file if it is available. Do the same for %% the .app.src file if it exists. -spec try_handle_resource_files(AppInfo, AppDir, ResourceFiles, valid | invalid | all) -> {true, AppInfo} | false when AppInfo :: rebar_app_info:t(), AppDir :: file:filename(), ResourceFiles :: [{app_resource_type(), file:filename()}]. try_handle_resource_files(AppInfo, AppDir, [{app, AppFile} | Rest], Validate) -> AppSrcFile = proplists:get_value(app_src, Rest), try_handle_app_file(AppInfo, AppDir, AppFile, AppSrcFile, Validate); try_handle_resource_files(AppInfo, AppDir, [{Type, AppSrcFile} | _Rest], Validate) when Type =:= app_src orelse Type =:= script -> try_handle_app_src_file(AppInfo, AppDir, AppSrcFile, Validate); try_handle_resource_files(AppInfo, AppDir, [{mix_exs, _MixExs} | Rest], Validate) -> %% prefer a rebar3 buildable app if both are found case try_handle_resource_files(AppInfo, AppDir, Rest, Validate) of false -> {true, rebar_app_info:project_type(AppInfo, mix)}; {true, _}=Result -> Result end; try_handle_resource_files(_AppInfo, _AppDir, [], _Validate) -> false. %% @doc Read in and parse the .app file if it is available. Do the same for %% the .app.src file if it exists. -spec try_handle_app_file(AppInfo, AppDir, File, AppSrcFile, valid | invalid | all) -> {true, AppInfo} | false when AppInfo :: rebar_app_info:t(), AppDir :: file:filename(), File :: file:filename(), AppSrcFile :: file:filename(). try_handle_app_file(AppInfo0, AppDir, File, AppSrcFile, Validate) -> try create_app_info(AppInfo0, AppDir, File) of AppInfo -> AppInfo1 = rebar_app_info:app_file(AppInfo, File), AppInfo2 = rebar_app_info:app_file_src(AppInfo1, AppSrcFile), case Validate of valid -> case rebar_app_utils:validate_application_info(AppInfo2) of true -> {true, AppInfo2}; _ -> false end; invalid -> case rebar_app_utils:validate_application_info(AppInfo2) of true -> false; _ -> {true, AppInfo2} end; all -> {true, AppInfo2} end catch throw:{error, {Module, Reason}} -> ?DEBUG("Falling back to app.src file because .app failed: ~ts", [Module:format_error(Reason)]), try_handle_app_src_file(AppInfo0, AppDir, AppSrcFile, Validate) end. %% @doc Read in the .app.src file if we aren't looking for a valid (already %% built) app. -spec try_handle_app_src_file(AppInfo, AppDir, AppSrcFile, valid | invalid | all) -> {true, AppInfo} | false when AppInfo :: rebar_app_info:t(), AppDir :: file:filename(), AppSrcFile :: file:filename(). try_handle_app_src_file(_AppInfo, _AppDir, undefined, valid) -> false; try_handle_app_src_file(_AppInfo, _AppDir, _AppSrcFile, valid) -> false; try_handle_app_src_file(AppInfo, AppDir, AppSrcFile, _) -> AppInfo1 = rebar_app_info:app_file(AppInfo, undefined), AppInfo2 = create_app_info(AppInfo1, AppDir, AppSrcFile), case filename:extension(AppSrcFile) of ".script" -> {true, rebar_app_info:app_file_src_script(AppInfo2, AppSrcFile)}; _ -> {true, rebar_app_info:app_file_src(AppInfo2, AppSrcFile)} end. %% @doc checks whether the given app is not blacklisted in the config. -spec enable(rebar_state:t(), rebar_app_info:t()) -> boolean(). enable(State, AppInfo) -> not lists:member(to_atom(rebar_app_info:name(AppInfo)), rebar_state:get(State, excluded_apps, [])). %% @private convert a binary to an atom. -spec to_atom(binary()) -> atom(). to_atom(Bin) -> list_to_atom(binary_to_list(Bin)). %% @private when looking for unknown apps, it's possible they have a %% rebar.config file specifying non-standard src_dirs. Check for a %% possible config file and extract src_dirs from it. find_config_src(AppDir, Default) -> case rebar_config:consult(AppDir) of [] -> {[], Default}; Terms -> %% TODO: handle profiles I guess, but we don't have that info {Terms, proplists:get_value(src_dirs, Terms, Default)} end.