Fossil

** Make arrangements to add one or more files or directories to the
** current checkout at the next commit.
**
** When adding files or directories recursively, filenames that begin
** with "." are excluded by default.  To include such files, add
** the "--dotfiles" option to the command-line.
**
** The --ignore and --clean options are comma-separate lists of glob patterns
** for files to be excluded.  Example:  '*.o,*.obj,*.exe'  If the --ignore
** option does not appear on the command line then the "ignore-glob" setting
** is used.  If the --clean option does not appear on the command line then
** the "clean-glob" setting is used.
**
** If files are attempted to be added explicitly on the command line which
** match "ignore-glob", a confirmation is asked first. This can be prevented
** using the -f|--force option.
**
** The --case-sensitive option determines whether or not filenames should
** be treated case sensitive or not. If the option is not given, the default
** depends on the global setting, or the operating system default, if not set.
**
** Options:
**
**    --case-sensitive <BOOL> Override the case-sensitive setting.
**    --dotfiles              include files beginning with a dot (".")
**    -f|--force              Add files without prompting
**    --ignore <CSG>          Ignore files matching patterns from the
**                            comma separated list of glob patterns.
**    --clean <CSG>           Also ignore files matching patterns from
**                            the comma separated list of glob patterns.
**
** See also: addremove, rm
*/
void add_cmd(void){
  int i;                     /* Loop counter */
  int vid;                   /* Currently checked out version */
  int nRoot;                 /* Full path characters in g.zLocalRoot */
  const char *zCleanFlag;    /* The --clean option or clean-glob setting */
  const char *zIgnoreFlag;   /* The --ignore option or ignore-glob setting */
  Glob *pIgnore, *pClean;    /* Ignore everything matching the glob patterns */
  unsigned scanFlags = 0;    /* Flags passed to vfile_scan() */
  int forceFlag;

  zCleanFlag = find_option("clean",0,1);
  zIgnoreFlag = find_option("ignore",0,1);
  forceFlag = find_option("force","f",0)!=0;
  if( find_option("dotfiles",0,0)!=0 ) scanFlags |= SCAN_ALL;

  /* We should be done with options.. */
  verify_all_options();

  db_must_be_within_tree();
  if( zCleanFlag==0 ){
    zCleanFlag = db_get("clean-glob", 0);
  }
  if( zIgnoreFlag==0 ){
    zIgnoreFlag = db_get("ignore-glob", 0);
  }
  if( db_get_boolean("dotfiles", 0) ) scanFlags |= SCAN_ALL;
  vid = db_lget_int("checkout",0);
  db_begin_transaction();
  db_multi_exec("CREATE TEMP TABLE sfile(x TEXT PRIMARY KEY %s)",
                filename_collation());
  pClean = glob_create(zCleanFlag);
  pIgnore = glob_create(zIgnoreFlag);
  nRoot = strlen(g.zLocalRoot);

  /* Load the names of all files that are to be added into sfile temp table */
  for(i=2; i<g.argc; i++){
    char *zName;
    int isDir;
    Blob fullName;

    /* file_tree_name() throws a fatal error if g.argv[i] is outside of the
    ** checkout. */
    file_tree_name(g.argv[i], &fullName, 0, 1);
    blob_reset(&fullName);

    file_canonical_name(g.argv[i], &fullName, 0);
    zName = blob_str(&fullName);
    isDir = file_wd_isdir(zName);
    if( isDir==1 ){
      vfile_scan(&fullName, nRoot-1, scanFlags, pClean, pIgnore);
    }else if( isDir==0 ){
      fossil_warning("not found: %s", zName);
    }else{
      char *zTreeName = &zName[nRoot];
      if( !forceFlag && glob_match(pIgnore, zTreeName) ){
        Blob ans;
        char cReply;

         "INSERT OR IGNORE INTO sfile(x) VALUES(%Q)",
         zTreeName
      );
    }
    blob_reset(&fullName);
  }
  glob_free(pIgnore);
  glob_free(pClean);

  add_files_in_sfile(vid);
  db_end_transaction(0);
}

/*
** This function adds a file to list of files to delete from disk after

** The command does not "commit".  You must run the "commit" separately
** as a separate step.
**
** Files and directories whose names begin with "." are ignored unless
** the --dotfiles option is used.
**
** The --ignore option overrides the "ignore-glob" setting, as do the
** --case-sensitive option with the "case-sensitive" setting and the
** --clean option with the "clean-glob" setting. See the documentation
** on the "settings" command for further information.
**
** The -n|--dry-run option shows what would happen without actually doing
** anything.
**
** This command can be used to track third party software.
**
** Options:
**   --case-sensitive <BOOL> Override the case-sensitive setting.
**   --dotfiles              Include files beginning with a dot (".")
**   --ignore <CSG>          Ignore files matching patterns from the
**                           comma separated list of glob patterns.
**   --clean <CSG>           Also ignore files matching patterns from
**                           the comma separated list of glob patterns.
**   -n|--dry-run            If given, display instead of run actions.
**
** See also: add, rm
*/
void addremove_cmd(void){
  Blob path;
  const char *zCleanFlag = find_option("clean",0,1);
  const char *zIgnoreFlag = find_option("ignore",0,1);
  unsigned scanFlags = find_option("dotfiles",0,0)!=0 ? SCAN_ALL : 0;
  int dryRunFlag = find_option("dry-run","n",0)!=0;
  int n;
  Stmt q;
  int vid;
  int nAdd = 0;
  int nDelete = 0;
  Glob *pIgnore, *pClean;

  if( !dryRunFlag ){
    dryRunFlag = find_option("test",0,0)!=0; /* deprecated */
  }

  /* We should be done with options.. */
  verify_all_options();

  /* Fail if unprocessed arguments are present, in case user expect the
  ** addremove command to accept a list of file or directory.
  */
  if( g.argc>2 ){
    fossil_fatal(
        "%s: Can only work on the entire checkout, no arguments supported.",
        g.argv[1]);
  }
  db_must_be_within_tree();
  if( zCleanFlag==0 ){
    zCleanFlag = db_get("clean-glob", 0);
  }
  if( zIgnoreFlag==0 ){
    zIgnoreFlag = db_get("ignore-glob", 0);
  }
  if( db_get_boolean("dotfiles", 0) ) scanFlags |= SCAN_ALL;
  vid = db_lget_int("checkout",0);
  db_begin_transaction();

  /* step 1:
  ** Populate the temp table "sfile" with the names of all unmanaged
  ** files currently in the check-out, except for files that match the
  ** --ignore or ignore-glob patterns and dot-files.  Then add all of
  ** the files in the sfile temp table to the set of managed files.
  */
  db_multi_exec("CREATE TEMP TABLE sfile(x TEXT PRIMARY KEY %s)",
                filename_collation());
  n = strlen(g.zLocalRoot);
  blob_init(&path, g.zLocalRoot, n-1);
  /* now we read the complete file structure into a temp table */
  pClean = glob_create(zCleanFlag);
  pIgnore = glob_create(zIgnoreFlag);
  vfile_scan(&path, blob_size(&path), scanFlags, pClean, pIgnore);
  glob_free(pIgnore);
  glob_free(pClean);
  nAdd = add_files_in_sfile(vid);

  /* step 2: search for missing files */
  db_prepare(&q,
      "SELECT pathname, %Q || pathname, deleted FROM vfile"
      " WHERE NOT deleted"
      " ORDER BY 1",

** are no files or directories named on the command-line, then add all
** unmanaged files anywhere in the checkout.
*/
static void locate_unmanaged_files(
  int argc,           /* Number of command-line arguments to examine */
  char **argv,        /* values of command-line arguments */
  unsigned scanFlags, /* Zero or more SCAN_xxx flags */
  Glob *pIgnore1,     /* Do not add files that match this GLOB */
  Glob *pIgnore2      /* Omit files matching this GLOB too */
){
  Blob name;   /* Name of a candidate file or directory */
  char *zName; /* Name of a candidate file or directory */
  int isDir;   /* 1 for a directory, 0 if doesn't exist, 2 for anything else */
  int i;       /* Loop counter */
  int nRoot;   /* length of g.zLocalRoot */

  db_multi_exec("CREATE TEMP TABLE sfile(x TEXT PRIMARY KEY %s)",
                filename_collation());
  nRoot = (int)strlen(g.zLocalRoot);
  if( argc==0 ){
    blob_init(&name, g.zLocalRoot, nRoot - 1);
    vfile_scan(&name, blob_size(&name), scanFlags, pIgnore1, pIgnore2);
    blob_reset(&name);
  }else{
    for(i=0; i<argc; i++){
      file_canonical_name(argv[i], &name, 0);
      zName = blob_str(&name);
      isDir = file_wd_isdir(zName);
      if( isDir==1 ){
        vfile_scan(&name, nRoot-1, scanFlags, pIgnore1, pIgnore2);
      }else if( isDir==0 ){
        fossil_warning("not found: %s", &zName[nRoot]);
      }else if( file_access(zName, R_OK) ){
        fossil_fatal("cannot open %s", &zName[nRoot]);
      }else{
        db_multi_exec(
           "INSERT OR IGNORE INTO sfile(x) VALUES(%Q)",

  /* We should be done with options.. */
  verify_all_options();

  if( zIgnoreFlag==0 ){
    zIgnoreFlag = db_get("ignore-glob", 0);
  }
  pIgnore = glob_create(zIgnoreFlag);
  locate_unmanaged_files(g.argc-2, g.argv+2, scanFlags, pIgnore, 0);
  glob_free(pIgnore);
  db_prepare(&q,
      "SELECT x FROM sfile"
      " WHERE x NOT IN (%s)"
      " ORDER BY 1",
      fossil_all_reserved_names(0)
  );

** files that are not officially part of the checkout. This operation
** cannot be undone. If one or more PATH arguments appear, then only
** the files named, or files contained with directories named, will be
** removed.
**
** Prompted are issued to confirm the removal of each file, unless
** the --force flag is used or unless the file matches glob pattern
** specified by the --clean option.  No file that matches glob patterns
** specified by --ignore or --keep will ever be deleted. The default
** values for --clean, --ignore, and --keep are determined by the
** (versionable) clean-glob, ignore-glob, and keep-glob settings.
** Files and subdirectories whose names begin with "." are automatically
** ignored unless the --dotfiles option is used.
**
** The --verily option ignores the keep-glob and ignore-glob settings
** and turns on --force, --dotfiles, and --emptydirs.  Use the --verily
** option when you really want to clean up everything.  Extreme care
** should be exercised when using the --verily option.
**
** Options:
**    --allckouts      Check for empty directories within any checkouts
**                     that may be nested within the current one.  This
**                     option should be used with great care because the

**                     prior to checking for any empty directories;
**                     therefore, directories that contain only files
**                     that were removed will be removed as well.
**    -f|--force       Remove files without prompting.
**    -i|--prompt      Prompt before removing each file.
**    -x|--verily      WARNING: Removes everything that is not a managed
**                     file or the repository itself.  This option
**                     implies the --force, --emptydirs, --dotfiles, and
**                     --disable-undo options.  Furthermore, it completely
**                     disregards the keep-glob and ignore-glob settings.
**                     However, it does honor the --ignore and --keep
**                     options.
**    --clean <CSG>    WARNING: Never prompt to delete any files matching
**                     this comma separated list of glob patterns.  Also,
**                     deletions of any files matching this pattern list
**                     cannot be undone.
**    --ignore <CSG>   Ignore files matching patterns from the
**                     comma separated list of glob patterns.
**    --keep <CSG>     Keep files matching this comma separated
**                     list of glob patterns.
**    -n|--dry-run     Delete nothing, but display what would have been
**                     deleted.
**    --no-prompt      This option disables prompting the user for input

  emptyDirsFlag = find_option("emptydirs","d",0)!=0 || dirsOnlyFlag;
  if( find_option("dotfiles",0,0)!=0 ) scanFlags |= SCAN_ALL;
  if( find_option("temp",0,0)!=0 ) scanFlags |= SCAN_TEMP;
  if( find_option("allckouts",0,0)!=0 ) scanFlags |= SCAN_NESTED;
  zIgnoreFlag = find_option("ignore",0,1);
  verboseFlag = find_option("verbose","v",0)!=0;
  zKeepFlag = find_option("keep",0,1);
  zCleanFlag = find_option("clean",0,1);
  db_must_be_within_tree();
  if( find_option("verily","x",0)!=0 ){
    verilyFlag = allFileFlag = allDirFlag = 1;
    emptyDirsFlag = 1;
    disableUndo = 1;
    scanFlags |= SCAN_ALL;
    zCleanFlag = 0;

  }
  if( zIgnoreFlag==0 && !verilyFlag ){
    zIgnoreFlag = db_get("ignore-glob", 0);
  }
  if( zKeepFlag==0 && !verilyFlag ){
    zKeepFlag = db_get("keep-glob", 0);
  }
  if( zCleanFlag==0 && !verilyFlag ){
    zCleanFlag = db_get("clean-glob", 0);
  }
  if( db_get_boolean("dotfiles", 0) ) scanFlags |= SCAN_ALL;
  verify_all_options();
  pIgnore = glob_create(zIgnoreFlag);
  pKeep = glob_create(zKeepFlag);
  pClean = glob_create(zCleanFlag);
  nRoot = (int)strlen(g.zLocalRoot);
  g.allowSymlinks = 1;  /* Find symlinks too */
  if( !dirsOnlyFlag ){
    Stmt q;
    Blob repo;
    if( !dryRunFlag && !disableUndo ) undo_begin();
    locate_unmanaged_files(g.argc-2, g.argv+2, scanFlags, pIgnore, 0);

    db_prepare(&q,
        "SELECT %Q || x FROM sfile"
        " WHERE x NOT IN (%s)"
        " ORDER BY 1",
        g.zLocalRoot, fossil_all_reserved_names(0)
    );
    if( file_tree_name(g.zRepositoryName, &repo, 0, 0) ){
      db_multi_exec("DELETE FROM sfile WHERE x=%B", &repo);
    }
    db_multi_exec("DELETE FROM sfile WHERE x IN (SELECT pathname FROM vfile)");
    while( db_step(&q)==SQLITE_ROW ){
      const char *zName = db_column_text(&q, 0);
      if( glob_match(pKeep, zName+nRoot) ){
        if( verboseFlag ){
          fossil_print("KEPT file \"%s\" not removed (due to --keep"
                       " or \"keep-glob\")\n", zName+nRoot);
        }
        continue;
      }
      if( !dryRunFlag && !glob_match(pClean, zName+nRoot) ){

        char *zPrompt = 0;
        char cReply;
        Blob ans = empty_blob;
        int undoRc = UNDO_NONE;
        if( alwaysPrompt ){
          zPrompt = mprintf("Remove unmanaged file \"%s\" (a=all/y/N)? ",
                            zName+nRoot);

    if( !dryRunFlag && !disableUndo ) undo_finish();
  }
  if( emptyDirsFlag ){
    Glob *pEmptyDirs = glob_create(db_get("empty-dirs", 0));
    Stmt q;
    Blob root;
    blob_init(&root, g.zLocalRoot, nRoot - 1);
    vfile_dir_scan(&root, blob_size(&root), scanFlags, pIgnore,
                   pEmptyDirs);
    blob_reset(&root);
    db_prepare(&q,
        "SELECT %Q || x FROM dscan_temp"
        " WHERE x NOT IN (%s) AND y = 0"
        " ORDER BY 1 DESC",
        g.zLocalRoot, fossil_all_reserved_names(0)
    );
    while( db_step(&q)==SQLITE_ROW ){
      const char *zName = db_column_text(&q, 0);
      if( glob_match(pKeep, zName+nRoot) ){
        if( verboseFlag ){
          fossil_print("KEPT directory \"%s\" not removed (due to --keep"
                       " or \"keep-glob\")\n", zName+nRoot);
        }
        continue;
      }
      if( !allDirFlag && !dryRunFlag && !glob_match(pClean, zName+nRoot) ){
        char cReply;
        if( !noPrompt ){
          Blob ans;
          char *prompt = mprintf("Remove empty directory \"%s\" (a=all/y/N)? ",
                                 zName+nRoot);
          prompt_user(prompt, &ans);
          cReply = blob_str(&ans)[0];
          fossil_free(prompt);
          blob_reset(&ans);
        }else{
          cReply = 'N';
        }
        if( cReply=='a' || cReply=='A' ){
          allDirFlag = 1;
        }else if( cReply!='y' && cReply!='Y' ){
          continue;
        }
      }
      if( dryRunFlag || file_rmdir(zName)==0 ){
        if( verboseFlag || dryRunFlag ){
          fossil_print("Removed unmanaged directory: %s\n", zName+nRoot);
        }
      }else if( verboseFlag ){
        fossil_print("Could not remove directory: %s\n", zName+nRoot);
      }
    }
    db_finalize(&q);
    glob_free(pEmptyDirs);
  }
  glob_free(pClean);
  glob_free(pKeep);
  glob_free(pIgnore);
}

/*
** Prompt the user for a check-in or stash comment (given in pPrompt),
** gather the response, then return the response in pComment.

  { "project-name",           CONFIGSET_PROJ },
  { "short-project-name",     CONFIGSET_PROJ },
  { "project-description",    CONFIGSET_PROJ },
  { "index-page",             CONFIGSET_PROJ },
  { "manifest",               CONFIGSET_PROJ },
  { "binary-glob",            CONFIGSET_PROJ },
  { "clean-glob",             CONFIGSET_PROJ },
  { "ignore-glob",            CONFIGSET_PROJ },
  { "keep-glob",              CONFIGSET_PROJ },
  { "crnl-glob",              CONFIGSET_PROJ },
  { "encoding-glob",          CONFIGSET_PROJ },
  { "empty-dirs",             CONFIGSET_PROJ },
  { "allow-symlinks",         CONFIGSET_PROJ },
  { "dotfiles",               CONFIGSET_PROJ },

  { "binary-glob",      0,             40, 1, 0, ""                    },
#if defined(_WIN32) || defined(__CYGWIN__) || defined(__DARWIN__) || \
    defined(__APPLE__)
  { "case-sensitive",   0,              0, 0, 0, "off"                 },
#else
  { "case-sensitive",   0,              0, 0, 0, "on"                  },
#endif
  { "clean-glob",       0,             40, 1, 0, ""                    },
  { "clearsign",        0,              0, 0, 0, "off"                 },
  { "crnl-glob",        0,             40, 1, 0, ""                    },
  { "default-perms",    0,             16, 0, 0, "u"                   },
  { "diff-binary",      0,              0, 0, 0, "on"                  },
  { "diff-command",     0,             40, 0, 0, ""                    },
  { "dont-push",        0,              0, 0, 0, "off"                 },
  { "dotfiles",         0,              0, 1, 0, "off"                 },

**                     for committing and merging purposes.  Example: *.jpg
**
**    case-sensitive   If TRUE, the files whose names differ only in case
**                     are considered distinct.  If FALSE files whose names
**                     differ only in case are the same file.  Defaults to
**                     TRUE for unix and FALSE for Cygwin, Mac and Windows.
**
**    clean-glob       The VALUE is a comma or newline-separated list of GLOB
**     (versionable)   patterns specifying files that the "clean" command will
**                     delete without prompting even when the -force flag has
**                     not been used.  Example:  *.a *.lib *.o
**
**    clearsign        When enabled, fossil will attempt to sign all commits
**                     with gpg.  When disabled (the default), commits will
**                     be unsigned.  Default: off
**
**    crnl-glob        A comma or newline-separated list of GLOB patterns for
**     (versionable)   text files in which it is ok to have CR, CR+NL or mixed
**                     line endings. Set to "*" to disable CR+NL checking.

**
**    https-login      Send login credentials using HTTPS instead of HTTP
**                     even if the login page request came via HTTP.
**
**    ignore-glob      The VALUE is a comma or newline-separated list of GLOB
**     (versionable)   patterns specifying files that the "add", "addremove",
**                     "clean", and "extra" commands will ignore.
**                     Example:  *.log customCode.c notes.txt
**
**    keep-glob        The VALUE is a comma or newline-separated list of GLOB
**     (versionable)   patterns specifying files that the "clean" command will
**                     keep.
**
**    localauth        If enabled, require that HTTP connections from
**                     127.0.0.1 be authenticated by password.  If
**                     false, all HTTP requests from localhost have
**                     unrestricted access to the repository.
**
**    main-branch      The primary branch for the project.  Default: trunk

{ "project-name",           CONFIGSET_PROJ },
{ "short-project-name",     CONFIGSET_PROJ },
{ "project-description",    CONFIGSET_PROJ },
{ "index-page",             CONFIGSET_PROJ },
{ "manifest",               CONFIGSET_PROJ },
{ "binary-glob",            CONFIGSET_PROJ },
{ "clean-glob",             CONFIGSET_PROJ },
{ "ignore-glob",            CONFIGSET_PROJ },
{ "keep-glob",              CONFIGSET_PROJ },
{ "crnl-glob",              CONFIGSET_PROJ },
{ "encoding-glob",          CONFIGSET_PROJ },
{ "empty-dirs",             CONFIGSET_PROJ },
{ "allow-symlinks",         CONFIGSET_PROJ },
{ "dotfiles",               CONFIGSET_PROJ },

  int n = 0;

  assert( g.db==0 );
  blob_init(&base, g.zRepositoryName, -1);
  sqlite3_open(":memory:", &g.db);
  db_multi_exec("CREATE TABLE sfile(x TEXT);");
  db_multi_exec("CREATE TABLE vfile(pathname);");
  vfile_scan(&base, blob_size(&base), 0, 0, 0);
  db_multi_exec("DELETE FROM sfile WHERE x NOT GLOB '*[^/].fossil'");
  n = db_int(0, "SELECT count(*) FROM sfile");
  if( n>0 ){
    Stmt q;
    @ <html>
    @ <head>
    @ <title>Repository List</title>

** are excluded from the scan.  Name matching occurs after the
** first nPrefix characters are elided from the filename.
*/
void vfile_scan(
  Blob *pPath,           /* Directory to be scanned */
  int nPrefix,           /* Number of bytes in directory name */
  unsigned scanFlags,    /* Zero or more SCAN_xxx flags */
  Glob *pIgnore1,        /* Do not add files that match this GLOB */
  Glob *pIgnore2         /* Omit files matching this GLOB too */
){
  DIR *d;
  int origSize;
  struct dirent *pEntry;
  int skipAll = 0;
  static Stmt ins;
  static int depth = 0;
  void *zNative;

  origSize = blob_size(pPath);
  if( pIgnore1 || pIgnore2 ){
    blob_appendf(pPath, "/");
    if( glob_match(pIgnore1, &blob_str(pPath)[nPrefix+1]) ) skipAll = 1;
    if( glob_match(pIgnore2, &blob_str(pPath)[nPrefix+1]) ) skipAll = 1;
    blob_resize(pPath, origSize);
  }
  if( skipAll ) return;

  if( depth==0 ){
    db_prepare(&ins,
       "INSERT OR IGNORE INTO sfile(x) SELECT :file"

        if( (scanFlags & SCAN_ALL)==0 ) continue;
        if( pEntry->d_name[1]==0 ) continue;
        if( pEntry->d_name[1]=='.' && pEntry->d_name[2]==0 ) continue;
      }
      zUtf8 = fossil_filename_to_utf8(pEntry->d_name);
      blob_appendf(pPath, "/%s", zUtf8);
      zPath = blob_str(pPath);
      if( glob_match(pIgnore1, &zPath[nPrefix+1]) ||
          glob_match(pIgnore2, &zPath[nPrefix+1]) ){
        /* do nothing */
#ifdef _DIRENT_HAVE_D_TYPE
      }else if( (pEntry->d_type==DT_UNKNOWN || pEntry->d_type==DT_LNK)
          ? (file_wd_isdir(zPath)==1) : (pEntry->d_type==DT_DIR) ){
#else
      }else if( file_wd_isdir(zPath)==1 ){
#endif
        if( !vfile_top_of_checkout(zPath) ){
          vfile_scan(pPath, nPrefix, scanFlags, pIgnore1, pIgnore2);
        }
#ifdef _DIRENT_HAVE_D_TYPE
      }else if( (pEntry->d_type==DT_UNKNOWN || pEntry->d_type==DT_LNK)
          ? (file_wd_isfile_or_link(zPath)) : (pEntry->d_type==DT_REG) ){
#else
      }else if( file_wd_isfile_or_link(zPath) ){
#endif