Fossil

Check-in [0ba08f9d26]
Login
Home Timeline Docs Branches Tickets Wiki DownloadMore...

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Faster determination of binary files, by not only checking for NUL <p>re-use looks_like_blob
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | improve_looks_like_binary
Files: files | file ages | folders
SHA1: 0ba08f9d26ecb3a53d79d05d3f95dfc8cc36c248
User & Date: jan.nijtmans 2012-10-30 20:10:28.744
Context
2012-10-31
08:43
Enhance looks_like_text(): <br>- Detect line-length overflow earlier, not at the next NL <br>- Implement the same binary and line-length check for UTF-16 as well <p>For UTF-16, the line-length limit is set to 2/3th of the line length limit for other text, because UTF-16 -> UTF-8 conversion can increase the line length (in bytes) by max 50%. This guarantees that a UTF-16 diff can be made by converting the two UTF-16 files to UTF-8 and then do a normal diff. check-in: 58702daa55 user: jan.nijtmans tags: improve_looks_like_binary
2012-10-30
20:10
Faster determination of binary files, by not only checking for NUL <p>re-use looks_like_blob check-in: 0ba08f9d26 user: jan.nijtmans tags: improve_looks_like_binary
18:14
Update to the version of SQLite that adds support for coroutines used to generate subqueries. check-in: bdbe6c74b8 user: drh tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/diff.c.
︙︙
187
188
189
190
191
192
193
194
195
196

197





198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221

**         delimited by carriage-return, line-feed pairs; however, the
**         encoding may not be UTF-8.
**
** (-2) -- The content appears to consist entirely of text, in the
**         UTF-16 (BE or LE) encoding.
*/
int looks_like_text(const Blob *pContent){
  const char *z = blob_buffer(pContent);
  unsigned int n = blob_size(pContent);
  int j, c;

  int result = 1;  /* Assume text with no CR/NL */






  /* Check individual lines.
  */
  if( n==0 ) return result;  /* Empty file -> text */
  c = *z;
  if( c==0 ) return 0;  /* \000 byte in a file -> binary */
  if ( n > 1 ){
    if ( (c==(char)0xff) && (z[1]==(char)0xfe) ){
      return -2;
    } else if ( (c==(char)0xfe) && (z[1]==(char)0xff) ){
      return -2;
    }
  }
  j = (c!='\n');
  while( --n>0 ){
    c = *++z; ++j;
    if( c==0 ) return 0;  /* \000 byte in a file -> binary */
    if( c=='\n' ){
      if( z[-1]=='\r' ){
        result = -1;  /* Contains CR/NL, continue */
      }
      if( j>LENGTH_MASK ){
        return 0;  /* Very long line -> binary */
      }








|

|
>

>
>
>
>
>





|

|

|






|








187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227

**         delimited by carriage-return, line-feed pairs; however, the
**         encoding may not be UTF-8.
**
** (-2) -- The content appears to consist entirely of text, in the
**         UTF-16 (BE or LE) encoding.
*/
int looks_like_text(const Blob *pContent){
  const unsigned char *z = blob_buffer(pContent);
  unsigned int n = blob_size(pContent);
  int j;
  unsigned char c;
  int result = 1;  /* Assume text with no CR/NL */
  static const char isBinary[256] = {
     1, 1, 1, 1,  1, 1, 1, 1,    1, 0, 0, 0,  0, 0, 1, 1,
     1, 1, 1, 1,  1, 1, 1, 1,    1, 1, 1, 0,  1, 1, 1, 1
  };


  /* Check individual lines.
  */
  if( n==0 ) return result;  /* Empty file -> text */
  c = *z;
  if( isBinary[c] ) return 0;  /* non-text byte in a file -> binary */
  if ( n > 1 ){
    if ( (c==0xff) && (z[1]==0xfe) ){
      return -2;
    } else if ( (c==0xfe) && (z[1]==0xff) ){
      return -2;
    }
  }
  j = (c!='\n');
  while( --n>0 ){
    c = *++z; ++j;
    if( isBinary[c] ) return 0;  /* \000 byte in a file -> binary */
    if( c=='\n' ){
      if( z[-1]=='\r' ){
        result = -1;  /* Contains CR/NL, continue */
      }
      if( j>LENGTH_MASK ){
        return 0;  /* Very long line -> binary */
      }
︙︙
Changes to src/doc.c.
︙︙
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

** For any other binary type, return "unknown/unknown".
*/
const char *mimetype_from_content(Blob *pBlob){
  int i;
  int n;
  const unsigned char *x;

  static const char isBinary[] = {
     1, 1, 1, 1,  1, 1, 1, 1,    1, 0, 0, 1,  0, 0, 1, 1,
     1, 1, 1, 1,  1, 1, 1, 1,    1, 1, 1, 0,  1, 1, 1, 1,
  };

  /* A table of mimetypes based on file content prefixes
  */
  static const struct {
    const char *zPrefix;       /* The file prefix */
    int size;                  /* Length of the prefix */
    const char *zMimetype;     /* The corresponding mimetype */
  } aMime[] = {
    { "GIF87a",                  6, "image/gif"  },
    { "GIF89a",                  6, "image/gif"  },
    { "\211PNG\r\n\032\n",       8, "image/png"  },
    { "\377\332\377",            3, "image/jpeg" },
    { "\377\330\377",            3, "image/jpeg" },
  };

  x = (const unsigned char*)blob_buffer(pBlob);
  n = blob_size(pBlob);
  for(i=0; i<n; i++){
    unsigned char c = x[i];
    if( c<=0x1f && isBinary[c] ){
      break;
    }
  }
  if( i>=n ){
    return 0;   /* Plain text */
  }
  for(i=0; i<sizeof(aMime)/sizeof(aMime[0]); i++){
    if( n>=aMime[i].size && memcmp(x, aMime[i].zPrefix, aMime[i].size)==0 ){
      return aMime[i].zMimetype;
    }
  }








<
<
<
<
<
















<
<
<
<
<
<
|








33
34
35
36
37
38
39





40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55






56
57
58
59
60
61
62
63

** For any other binary type, return "unknown/unknown".
*/
const char *mimetype_from_content(Blob *pBlob){
  int i;
  int n;
  const unsigned char *x;






  /* A table of mimetypes based on file content prefixes
  */
  static const struct {
    const char *zPrefix;       /* The file prefix */
    int size;                  /* Length of the prefix */
    const char *zMimetype;     /* The corresponding mimetype */
  } aMime[] = {
    { "GIF87a",                  6, "image/gif"  },
    { "GIF89a",                  6, "image/gif"  },
    { "\211PNG\r\n\032\n",       8, "image/png"  },
    { "\377\332\377",            3, "image/jpeg" },
    { "\377\330\377",            3, "image/jpeg" },
  };

  x = (const unsigned char*)blob_buffer(pBlob);
  n = blob_size(pBlob);






  if( looks_like_text(pBlob) ){
    return 0;   /* Plain text */
  }
  for(i=0; i<sizeof(aMime)/sizeof(aMime[0]); i++){
    if( n>=aMime[i].size && memcmp(x, aMime[i].zPrefix, aMime[i].size)==0 ){
      return aMime[i].zMimetype;
    }
  }
︙︙
This page was generated in about 0.027s by Fossil 2.26 [1205ec86cb] 2025-04-30 16:57:32