/*****************************************************************************/
/*
                                 Descr.c

This module implements a full multi-threaded, AST-driven, asynchronous
description generator for a file.

This is not a full task in the sense of a file transfer.  This module merely
determines a file description.  For an HTML file this is by looking for the
contents of a <title></title> or <h2></h2> tag pair.  For a plain text file,
the first non-blank line.  If no internal description can be determined the
buffer is made an empty string.

This module never returns a valid status and ALWAYS calls the supplied next
task (AST) function.  This should check for a generated error message to
determine is there were any problems.


VERSION HISTORY
---------------
06-OCT-2010  MGD  DescriptionError() !&S to %X!8XL
05-OCT-2002  MGD  refine VMS security profile usage
27-APR-2002  MGD  make SYSPRV enabled ASTs asynchronous
04-AUG-2001  MGD  support module WATCHing
02-JAN-2000  MGD  no significant modifications for ODS-5 (no NAM block)
20-NOV-1999  MGD  increase buffer size to 2048 (in part to allow for longer
                  "lines", also to accomodate fixed, 512 byte, FTPed records). 
19-SEP-1998  MGD  improve granularity of file open, connect, close
25-OCT-1997  MGD  DescriptionEscape() now includes only printable characters
17-AUG-1997  MGD  message database,
                  SYSUAF-authenticated users security-profile
01-FEB-1997  MGD  HTTPd version 4
23-MAY-1996  MGD  functionality generalized, moved from directory module
*/
/*****************************************************************************/

#ifdef WASD_VMS_V7
#undef _VMS__V6__SOURCE
#define _VMS__V6__SOURCE
#undef __VMS_VER
#define __VMS_VER 70000000
#undef __CRTL_VER
#define __CRTL_VER 70000000
#endif

/* standard C header files */
#include <ctype.h>
#include <stdio.h>
#include <string.h>

/* VMS related header files */
#include <atrdef.h>
#include <descrip.h>
#include <dvidef.h>
#include <fibdef.h>
#include <iodef.h>
#include <rmsdef.h>
#include <rms.h>
#include <ssdef.h>
#include <stsdef.h>

/* application related header files */
#include "wasd.h"

#define WASD_MODULE "DESCR"

/********************/
/* external storage */
/********************/

extern char  SoftwareID[];

extern CONFIG_STRUCT  Config;
extern MSG_STRUCT  Msgs;
extern WATCH_STRUCT  Watch;

/*****************************************************************************/
/*
*/

Description
(
REQUEST_STRUCT *rqptr,
REQUEST_AST NextTaskFunction,
char *FileName,
char *DescriptionBuffer,
int SizeOfDescriptionBuffer,
int ContentTypes
)
{
   int  status;
   char  *cptr, *sptr, *zptr;
   DESCR_TASK  *tkptr;

   /*********/
   /* begin */
   /*********/

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER,
                 "Description() !&A !AZ", NextTaskFunction, FileName);

   if (ERROR_REPORTED (rqptr))
   {
      /* previous error, cause threaded processing to unravel */
      SysDclAst (NextTaskFunction, rqptr);
      return;
   }

   /* set up the task structure (possibly multiple serially) */
   if (rqptr->DescrTaskPtr)
   {
      tkptr = rqptr->DescrTaskPtr;
      memset (tkptr, 0, sizeof(DESCR_TASK));
   }
   else
   {
      rqptr->DescrTaskPtr = tkptr = (DESCR_TASK*)
         VmGetHeap (rqptr, sizeof(DESCR_TASK));
   }
   tkptr->NextTaskFunction = NextTaskFunction;

   /* setup using the calling function's buffer space */
   *(tkptr->DescriptionBufferPtr = DescriptionBuffer) = '\0';
   tkptr->SizeOfDescriptionBuffer = SizeOfDescriptionBuffer;
   /* indicate no description could be generated */
   tkptr->DescriptionBufferPtr[0] = DESCRIPTION_IMPOSSIBLE;

   cptr = FileName;
   zptr = (sptr = tkptr->FileName) + sizeof(tkptr->FileName);
   while (*cptr && sptr < zptr) *sptr++ = *cptr++;
   if (sptr >= zptr)
   {
      ErrorGeneralOverflow (rqptr, FI_LI);
      DescriptionEnd (rqptr);
      return;
   }
   *sptr = '\0';
   tkptr->FileNameLength = sptr - tkptr->FileName;

   /***************/
   /* description */
   /***************/

   /* using the pointer from above find the file type */
   while (sptr > tkptr->FileName && *sptr != '.') sptr--;
   cptr = ConfigContentType (NULL, sptr);

   if (to_lower(cptr[0]) == 't' &&
       ConfigSameContentType (cptr, "text/", 5))
   {
      if ((ContentTypes & DESCRIPTION_TEXT_HTML ||
           ContentTypes & DESCRIPTION_ALL) &&
          ConfigSameContentType (cptr, "text/html", -1))
      {
         tkptr->TextHtmlFile = true;
         DescriptionOpen (rqptr);
         return;
      }
      else
      if ((ContentTypes & DESCRIPTION_TEXT_PLAIN ||
           ContentTypes & DESCRIPTION_ALL) &&
          ConfigSameContentType (cptr, "text/plain", -1))
      {
         tkptr->TextHtmlFile = false;
         DescriptionOpen (rqptr);
         return;
      }
   }

   /* declare the next task */
   SysDclAst (tkptr->NextTaskFunction, rqptr);
}

/*****************************************************************************/
/*
*/ 

DescriptionEnd (REQUEST_STRUCT *rqptr)

{
   int  status;
   DESCR_TASK  *tkptr;

   /*********/
   /* begin */
   /*********/

   /* retrieve the pointer to the task structure */
   tkptr = rqptr->DescrTaskPtr;

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER,
                 "DescriptionEnd() !&F !AZ", &DescriptionEnd,
                 tkptr->DescriptionBufferPtr);

   if (tkptr->FileFab.fab$w_ifi)
   {
      tkptr->FileFab.fab$l_fop |= FAB$M_ASY;
      sys$close (&tkptr->FileFab, &DescriptionCloseAst, &DescriptionCloseAst);
   }
   else
      SysDclAst (tkptr->NextTaskFunction, rqptr);
}

/*****************************************************************************/
/*
AST delivered after asynchronous close in DescriptionEnd().
*/

DescriptionCloseAst (struct FAB *FabPtr)

{
   /*********/
   /* begin */
   /*********/

   if (WATCH_MOD)
   {
      REQUEST_STRUCT  *rqptr;
      rqptr = FabPtr->fab$l_ctx;
      if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
         WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER,
"DescriptionCloseAst() !&F sts!&S stv:!&S", &DescriptionCloseAst,
                    FabPtr->fab$l_sts, FabPtr->fab$l_stv);
   }

   DescriptionEnd (FabPtr->fab$l_ctx);
} 

/*****************************************************************************/
/*
*/

DescriptionOpen (REQUEST_STRUCT *rqptr)

{
   int  status;
   DESCR_TASK  *tkptr;

   /*********/
   /* begin */
   /*********/

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER,
                 "DescriptionOpen() !&F !AZ !UL", &DescriptionOpen, 
                 rqptr->DescrTaskPtr->FileName,
                 rqptr->DescrTaskPtr->TextHtmlFile);

   tkptr = rqptr->DescrTaskPtr;

   tkptr->FileFab = cc$rms_fab;
   tkptr->FileFab.fab$l_ctx = rqptr;
   tkptr->FileFab.fab$b_fac = FAB$M_GET;
   tkptr->FileFab.fab$l_fna = tkptr->FileName;  
   tkptr->FileFab.fab$b_fns = tkptr->FileNameLength;
   tkptr->FileFab.fab$b_shr = FAB$M_SHRGET;

   AuthAccessEnable (rqptr, tkptr->FileName, AUTH_ACCESS_READ);

   tkptr->FileFab.fab$l_fop |= FAB$M_ASY;
   sys$open (&tkptr->FileFab, &DescriptionOpenAst, &DescriptionOpenAst);

   AuthAccessEnable (rqptr, 0, 0);
} 

/*****************************************************************************/
/*
AST called from DescriptionOpen() when asynchronous open completes. Initiate
an asynchronous RAB connect.
*/

DescriptionOpenAst (struct FAB *FabPtr)

{
   int  status,
        MultiBlockCount;
   REQUEST_STRUCT  *rqptr;
   DESCR_TASK  *tkptr;

   /*********/
   /* begin */
   /*********/

   rqptr = FabPtr->fab$l_ctx;

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER,
"DescriptionOpenAst() !&F sts:!&S stv:!&S", &DescriptionOpenAst,
                 FabPtr->fab$l_sts, FabPtr->fab$l_stv);

#if WATCH_MOD
   HttpdCheckPriv (FI_LI);
#endif /* WATCH_MOD */

   tkptr = rqptr->DescrTaskPtr;

   if (VMSnok (status = FabPtr->fab$l_sts))
   {
      DescriptionError (rqptr, status, __LINE__);
      DescriptionEnd (rqptr);
      return;
   }

   tkptr->FileRab = cc$rms_rab;
   tkptr->FileRab.rab$l_ctx = rqptr;
   tkptr->FileRab.rab$l_fab = &tkptr->FileFab;
   /* 2 buffers and read ahead performance option */
   tkptr->FileRab.rab$b_mbf = 2;
   tkptr->FileRab.rab$l_rop = RAB$M_RAH | RAB$M_ASY;
   tkptr->FileRab.rab$l_ubf = tkptr->ReadBuffer;
   tkptr->FileRab.rab$w_usz = sizeof(tkptr->ReadBuffer)-1;

   sys$connect (&tkptr->FileRab, &DescriptionConnectAst,
                                 &DescriptionConnectAst);
}

/*****************************************************************************/
/*
AST called from FileOpenAst() when asynchronous RAB connect completes.
Initiate stream-LF file conversion if the file meets requirements.  If
suitable for caching then initiate a cache load and begin reading the file
contents.
*/ 

DescriptionConnectAst (struct RAB *RabPtr)

{
   int  status;
   REQUEST_STRUCT  *rqptr;
   DESCR_TASK  *tkptr;

   /*********/
   /* begin */
   /*********/

   rqptr = RabPtr->rab$l_ctx;

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER,
"DescriptionConnectAst() !&F sts:!&S stv:!&S", &DescriptionConnectAst,
                 RabPtr->rab$l_sts, RabPtr->rab$l_stv);

   tkptr = rqptr->DescrTaskPtr;

   if (VMSnok (status = RabPtr->rab$l_sts))
   {
      DescriptionError (rqptr, status, __LINE__);
      DescriptionEnd (rqptr);
      return;
   }

   *(tkptr->DescriptionPtr = tkptr->DescriptionBufferPtr) = '\0';
   tkptr->DescriptionRetrieved = false;
   tkptr->DescriptionLineCount = 0;

   /* queue up a read of the first record in the file */
   if (tkptr->TextHtmlFile)
      sys$get (&tkptr->FileRab, &DescriptionHtmlRecord,
                                &DescriptionHtmlRecord);
   else
      sys$get (&tkptr->FileRab, &DescriptionPlainRecord,
                                &DescriptionPlainRecord);
}

/*****************************************************************************/
/*
This is an AST completion routine called each time sys$get() completes.  It 
returns with the next record, end-of-file, or some other error.  Examine the 
record looking for text comprising the first of either the <title></title> 
element or a heading element (e.g. <h2></h2>), retrieving this and using it 
as the file description.
*/

DescriptionHtmlRecord (struct RAB *RabPtr)

{
   int  status;
   char  *dptr, *rptr, *zptr;
   REQUEST_STRUCT  *rqptr;
   DESCR_TASK  *tkptr;

   /*********/
   /* begin */
   /*********/

   rqptr = RabPtr->rab$l_ctx;

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER,
"DescriptionHtmlRecord() !&F sts:!&S stv:!&S !UL", &DescriptionHtmlRecord,
                 RabPtr->rab$l_sts, RabPtr->rab$l_stv, RabPtr->rab$w_rsz);

   tkptr = rqptr->DescrTaskPtr;

   if (VMSnok (tkptr->FileRab.rab$l_sts))
   {
      if (tkptr->FileRab.rab$l_sts == RMS$_EOF)
      {
         /***************/
         /* end-of-file */
         /***************/

         DescriptionEnd (rqptr);
         return;
      }

      /**********************/
      /* error reading file */
      /**********************/

      DescriptionError (rqptr, tkptr->FileRab.rab$l_sts, __LINE__);
      DescriptionEnd (rqptr);
      return;
   }

   tkptr->DescriptionLineCount++;
   tkptr->FileRab.rab$l_ubf[tkptr->FileRab.rab$w_rsz] = '\0';

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchDataDump (tkptr->FileRab.rab$l_ubf, tkptr->FileRab.rab$w_rsz);

   /****************************************/
   /* look for the <title> or <Hn> element */
   /****************************************/

   /* 
      'zptr' points at the last character of the description storage.
      It prevents non-closed <title> or <Hn> structures from running off
      the end of the storage area.
   */

   dptr = tkptr->DescriptionPtr;
   zptr = tkptr->DescriptionBufferPtr + tkptr->SizeOfDescriptionBuffer - 1;

   /*
      Add a space for any record boundary occuring in the title,
      except if it's a leading space (i.e. first after the leading quote).
   */
   if (tkptr->DescriptionInside &&
       dptr > tkptr->DescriptionBufferPtr+1 &&
       dptr < zptr)
      *dptr++ = ' ';

   for (rptr = tkptr->FileRab.rab$l_ubf; *rptr; rptr++)
   {
      if (*rptr == '<')
      {
         tkptr->DescriptionOpeningTagCount++;
         if (to_upper(rptr[1]) == 'T' && strsame (rptr+2, "ITLE>", 5))
         {
            tkptr->DescriptionInside = true;
            rptr += 6;
            tkptr->DescriptionClosingTagCount++;
            continue;
         }
         if (to_upper(rptr[1]) == 'H' && isdigit(rptr[2]) && rptr[3] == '>')
         {
            tkptr->DescriptionInside = true;
            rptr += 3;
            tkptr->DescriptionClosingTagCount++;
            continue;
         }
         if (rptr[1] == '/')
         {
            if (to_upper(rptr[2]) == 'T' && strsame (rptr+3, "ITLE>", 5))
            {
               tkptr->DescriptionInside = false;
               tkptr->DescriptionRetrieved = true;
               rptr += 7;
               tkptr->DescriptionClosingTagCount++;
               /* suppress any trailing white-space */
               if (dptr > tkptr->DescriptionBufferPtr+1) dptr--;
               while (dptr > tkptr->DescriptionBufferPtr+1 &&
                      ISLWS(*dptr)) dptr--;
               *++dptr = '\0';
               continue;
            }
            if (to_upper(rptr[2]) == 'H' && isdigit(rptr[3]) && rptr[4] == '>')
            {
               tkptr->DescriptionInside = false;
               tkptr->DescriptionRetrieved = true;
               rptr += 4;
               tkptr->DescriptionClosingTagCount++;
               /* suppress any trailing white-space */
               if (dptr > tkptr->DescriptionBufferPtr+1) dptr--;
               while (dptr > tkptr->DescriptionBufferPtr+1 &&
                      ISLWS(*dptr)) dptr--;
               *++dptr = '\0';
               continue;
            }
         }
      }
      else
      if (*rptr == '>')
      {
         tkptr->DescriptionClosingTagCount++;
         continue;
      }

      /* don't scan the line further if we've found what we're looking for */
      if (tkptr->DescriptionRetrieved) break;

      /* if we're currently inside a tag name (between "<" and ">") */
      if (tkptr->DescriptionOpeningTagCount > tkptr->DescriptionClosingTagCount)
         continue;

      /* suppress leading white-space */
      if (tkptr->DescriptionInside &&
          !(dptr == tkptr->DescriptionBufferPtr && ISLWS(*rptr)) &&
          isprint (*rptr) &&
          dptr < zptr)
         *dptr++ = *rptr;
   }

   /* terminate the title/description */
   if (dptr > tkptr->DescriptionBufferPtr) *dptr = '\0';
   tkptr->DescriptionPtr = dptr;

   if (tkptr->DescriptionRetrieved ||
       tkptr->DescriptionLineCount > Config.cfDir.DescriptionLines)
   {
      /************************************/
      /* title found, or out-of-patience! */
      /************************************/

      DescriptionEnd (rqptr);
      return;
   }
   else
   {
      /*****************/
      /* still looking */
      /*****************/

      /* queue another read, completion AST back to this function again */
      sys$get (&tkptr->FileRab, &DescriptionHtmlRecord,
                                &DescriptionHtmlRecord);
   }
}

/*****************************************************************************/
/*
This is an AST completion routine called each time sys$get() completes.  It 
returns with the next record, end-of-file, or some other error.
*/

DescriptionPlainRecord (struct RAB *RabPtr)

{
   int  status;
   char  *dptr, *rptr, *zptr;
   REQUEST_STRUCT  *rqptr;
   DESCR_TASK  *tkptr;

   /*********/
   /* begin */
   /*********/

   rqptr = RabPtr->rab$l_ctx;

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER,
"DescriptionPlainRecord() !&F sts:!&S stv:!&S !UL", &DescriptionPlainRecord,
                 RabPtr->rab$l_sts, RabPtr->rab$l_stv, RabPtr->rab$w_rsz);

   tkptr = rqptr->DescrTaskPtr;

   if (VMSnok (tkptr->FileRab.rab$l_sts))
   {
      if (tkptr->FileRab.rab$l_sts == RMS$_EOF)
      {
         /***************/
         /* end-of-file */
         /***************/

         DescriptionEnd (rqptr);
         return;
      }

      /**********************/
      /* error reading file */
      /**********************/

      {
         DescriptionError (rqptr, tkptr->FileRab.rab$l_sts, __LINE__);
         DescriptionEnd (rqptr);
         return;
      }
   }

   tkptr->DescriptionLineCount++;
   tkptr->FileRab.rab$l_ubf[tkptr->FileRab.rab$w_rsz] = '\0';

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchDataDump (tkptr->FileRab.rab$l_ubf, tkptr->FileRab.rab$w_rsz);

   /************************************/
   /* look for first plain-text record */
   /************************************/

   for (rptr = tkptr->FileRab.rab$l_ubf; *rptr && !isalnum(*rptr); rptr++);
   if (!isalnum(*rptr))
   {
      if (tkptr->DescriptionLineCount <= Config.cfDir.DescriptionLines)
      {
         /* queue another read, completion AST back to this function again */
         sys$get (&tkptr->FileRab, &DescriptionPlainRecord,
                                   &DescriptionPlainRecord);
         return;
      }

      /********************/
      /* out-of-patience! */
      /********************/

      DescriptionEnd (rqptr);
      return;
   }

   /*******************/
   /* get description */
   /*******************/

   dptr = tkptr->DescriptionBufferPtr;
   zptr = tkptr->DescriptionBufferPtr + tkptr->SizeOfDescriptionBuffer - 1;
   while (*rptr && dptr < zptr) *dptr++ = *rptr++;
   *dptr = '\0';

   DescriptionEscape (rqptr);

   DescriptionEnd (rqptr);
}

/*****************************************************************************/
/*
Make sure we only meaningful and HTML-acceptable characters into the
description.
*/

DescriptionEscape (REQUEST_STRUCT *rqptr)

{
   BOOL  Changed;
   char  *cptr, *sptr, *zptr;
   DESCR_TASK  *tkptr;
   char  Buffer [256];

   /*********/
   /* begin */
   /*********/

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER, "DescriptionEscape()");

   /* retrieve the pointer to the task structure */
   tkptr = rqptr->DescrTaskPtr;

   Changed = false;
   if (tkptr->SizeOfDescriptionBuffer > sizeof(Buffer))
      zptr = (sptr = Buffer) + tkptr->SizeOfDescriptionBuffer - 1;
   else
      zptr = (sptr = Buffer) + sizeof(Buffer)-1;
   for (cptr = tkptr->DescriptionBufferPtr; *cptr && sptr+6 < zptr; cptr++)
   {
      switch (*cptr)
      {
         case '<' :
            memcpy (sptr, "&lt;", 4); sptr += 4; Changed = true; break;
         case '>' :
            memcpy (sptr, "&gt;", 4); sptr += 4; Changed = true; break;
         case '&' :
            memcpy (sptr, "&amp;", 5); sptr += 5; Changed = true; break;
         case '\"' :
            *sptr++ = '\''; Changed = true; break; 
         default:
            if (isprint(*cptr)) *sptr++ = *cptr;
      } 
   }
   *sptr++ = '\0';
   if (Changed) memcpy (tkptr->DescriptionBufferPtr, Buffer, sptr - Buffer);
}

/*****************************************************************************/
/*
Put the hexadecimal VMS status value into the description buffer as an error
indication.
*/

DescriptionError
(
REQUEST_STRUCT *rqptr,
int StatusValue,
int SourceLineNumber
)
{
   static  $DESCRIPTOR (ErrorFaoDsc,
           "<font color=\"#ff0000\">[Error %X!8XL]</font><!!-- !UL -->\0");
   static  $DESCRIPTOR (BufferDsc, "");

   DESCR_TASK  *tkptr;

   /*********/
   /* begin */
   /*********/

   if (WATCHMOD (rqptr, WATCH_MOD__OTHER))
      WatchThis (WATCHITM(rqptr), WATCH_MOD__OTHER,
                 "DescriptionError() !&S", StatusValue);

   tkptr = rqptr->DescrTaskPtr;

   BufferDsc.dsc$a_pointer = tkptr->DescriptionBufferPtr;
   BufferDsc.dsc$w_length = tkptr->SizeOfDescriptionBuffer;
   sys$fao (&ErrorFaoDsc, 0, &BufferDsc, StatusValue, SourceLineNumber);
}

/*****************************************************************************/