Fossil

#if INTERFACE

/*
** This macro is designed to return non-zero if the specified blob contains
** data that MAY be binary in nature; otherwise, zero will be returned.
*/
#define looks_like_binary(blob) \
    ((looks_like_utf8((blob), LOOK_BINARY, 0) & LOOK_BINARY) != LOOK_NONE)

/*
** Output flags for the looks_like_utf8() and looks_like_utf16() routines used
** to convey status information about the blob content.
*/
#define LOOK_NONE    ((int)0x00000000) /* Nothing special was found. */
#define LOOK_NUL     ((int)0x00000001) /* One or more NUL chars were found. */

** carriage-return, and line-feed.
**
** This function examines the contents of the blob until one of the flags
** specified in "stopFlags" is set.
**
************************************ WARNING **********************************
*/
int looks_like_utf8(const Blob *pContent, int stopFlags, int fVerbose){
  const char *z = blob_buffer(pContent);
  unsigned int n = blob_size(pContent);
  int j, c, flags = LOOK_NONE;  /* Assume UTF-8 text, prove otherwise */
  int nLine = 1;

  if( n==0 ) return flags;  /* Empty file -> text */
  c = *z;
  if( c==0 ){
    flags |= LOOK_NUL;  /* NUL character in a file -> binary */
    if( fVerbose ) fossil_print("NUL at start\n");
  }else if( c=='\r' ){
    flags |= LOOK_CR;
    if( fVerbose ) fossil_print("CR at start\n");
    if( n<=1 || z[1]!='\n' ){
      flags |= LOOK_LONE_CR;  /* Not enough chars or next char not LF */
      if( fVerbose ) fossil_print("Lone CR at start\n");
    }
  }
  j = (c!='\n');
  if( !j ) flags |= (LOOK_LF | LOOK_LONE_LF);  /* Found LF as first char */
  while( !(flags&stopFlags) && --n>0 ){
    int c2 = c;
    c = *++z; ++j;
    if( c==0 ){
      if( fVerbose && !(flags&LOOK_NUL) ){
        fossil_print("NUL on line %d\n", nLine);
      }
      flags |= LOOK_NUL;  /* NUL character in a file -> binary */
    }else if( c=='\n' ){
      flags |= LOOK_LF;
      if( c2=='\r' ){
        if( fVerbose && !(flags&LOOK_CRLF) ){
          fossil_print("CRLF on line %d\n", nLine);
        }
        flags |= (LOOK_CR | LOOK_CRLF);  /* Found LF preceded by CR */
      }else{
        if( fVerbose && !(flags&LOOK_LONE_LF) ){
          fossil_print("Lone LF on line %d\n", nLine);
        }
        flags |= LOOK_LONE_LF;
      }
      if( j>LENGTH_MASK ){
        if( fVerbose && !(flags&LOOK_LONG) ){
          fossil_print("Line %d is longer than %d bytes\n", nLine, j);
        }
        flags |= LOOK_LONG;  /* Very long line -> binary */
      }
      ++nLine;
      j = 0;
    }else if( c=='\r' ){
      flags |= LOOK_CR;
      if( n<=1 || z[1]!='\n' ){
        if( fVerbose && !(flags&LOOK_LONE_CR) ){
          fossil_print("Lone CR on line %d\n", nLine);
        }
        flags |= LOOK_LONE_CR;  /* Not enough chars or next char not LF */
      }
    }
  }
  if( n ){
    flags |= LOOK_SHORT;  /* The whole blob was not examined */
  }

** Usage:  %fossil test-looks-like-utf FILENAME
**
** Options:
**    -n|--limit N     Repeat looks-like function N times, for
**                     performance measurement. Default = 1
**    --utf8           Ignoring BOM and file size, force UTF-8 checking
**    --utf16          Ignoring BOM and file size, force UTF-16 checking
**    -v|--verbose     Report the line numbers where each flag is first set
**
** FILENAME is the name of a file to check for textual content in the UTF-8
** and/or UTF-16 encodings.
*/
void looks_like_utf_test_cmd(void){
  Blob blob;         /* the contents of the specified file */
  int fUtf8 = 0;     /* return value of starts_with_utf8_bom() */
  int fUtf16 = 0;    /* return value of starts_with_utf16_bom() */
  int fUnicode = 0;  /* return value of could_be_utf16() */
  int lookFlags = 0; /* output flags from looks_like_utf8/utf16() */
  int bRevUtf16 = 0; /* non-zero -> UTF-16 byte order reversed */
  int fForceUtf8 = find_option("utf8",0,0)!=0;
  int fForceUtf16 = find_option("utf16",0,0)!=0;
  const char *zCount = find_option("limit","n",1);
  int fVerbose = find_option("verbose","v",0)!=0;
  int nRepeat = 1;

  if( g.argc!=3 ) usage("FILENAME");
  if( zCount ){
    nRepeat = atoi(zCount);
  }
  blob_read_from_file(&blob, g.argv[2], ExtFILE);
  while( --nRepeat >= 0 ){
    fUtf8 = starts_with_utf8_bom(&blob, 0);
    fUtf16 = starts_with_utf16_bom(&blob, 0, &bRevUtf16);
    if( fForceUtf8 ){
      fUnicode = 0;
    }else{
      fUnicode = could_be_utf16(&blob, 0) || fForceUtf16;
    }
    if( fUnicode ){
      lookFlags = looks_like_utf16(&blob, bRevUtf16, 0);
    }else{
      lookFlags = looks_like_utf8(&blob, 0, fVerbose) | invalid_utf8(&blob);
    }
  }
  fossil_print("File \"%s\" has %d bytes.\n",g.argv[2],blob_size(&blob));
  fossil_print("Starts with UTF-8 BOM: %s\n",fUtf8?"yes":"no");
  fossil_print("Starts with UTF-16 BOM: %s\n",
               fUtf16?(bRevUtf16?"reversed":"yes"):"no");
  fossil_print("Looks like UTF-%s: %s\n",fUnicode?"16":"8",