Sitefinity search can't find the words inside your PDFs, PowerPoints or spreadsheets

<?xml version="1.0" encoding="utf-8"?>
<!--
  App_Data/Sitefinity/Configuration/DocumentServiceConfig.config

  Sitefinity MERGES these with its built-in registrations rather than replacing them, so listing
  only what you are adding or overriding is enough. Out of the box it registers exactly five mime
  types (pdf, docx, html, plain text, rtf); everything else indexes title-only.

  The type string is "Namespace.ClassName, AssemblyName" with no version or public key token.
  Replace "YourAssembly" with the assembly your extractors are compiled into.

  Two things that catch people out:

    1. Registering an extractor does NOTHING to documents already in the index. Deploy the
       assembly, add this config, then run a full reindex from
       Administration > Search indexes > (your index) > Reindex.

    2. There is one registration per mime type, not per extractor. PresentationDocument opens
       pptx, pptm and ppsx identically, but each mime type still needs its own line, and
       Sitefinity constructs a separate instance for each and tells it which mime it is via
       Initialize(mimeType, config).
-->
<documentServiceConfig>
  <extractorSettings>

    <!-- Overrides Sitefinity's DefaultPdfTextExtractor, which cannot open PDFs whose structure
         tree throws during import. Same mime type, so this replaces the built-in entry. -->
    <add mimeType="application/pdf"
         extractorType="Sitefinity.TextExtractors.PdfTextExtractor, YourAssembly" />

    <!-- PowerPoint: pptx, pptm, ppsx. No built-in extractor exists for any of these. -->
    <add mimeType="application/vnd.openxmlformats-officedocument.presentationml.presentation"
         extractorType="Sitefinity.TextExtractors.PptxTextExtractor, YourAssembly" />
    <add mimeType="application/vnd.ms-powerpoint.presentation.macroEnabled.12"
         extractorType="Sitefinity.TextExtractors.PptxTextExtractor, YourAssembly" />
    <add mimeType="application/vnd.openxmlformats-officedocument.presentationml.slideshow"
         extractorType="Sitefinity.TextExtractors.PptxTextExtractor, YourAssembly" />

    <!-- Excel: xlsx, xlsm, xltx. No built-in extractor exists for any of these. -->
    <add mimeType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
         extractorType="Sitefinity.TextExtractors.XlsxTextExtractor, YourAssembly" />
    <add mimeType="application/vnd.ms-excel.sheet.macroEnabled.12"
         extractorType="Sitefinity.TextExtractors.XlsxTextExtractor, YourAssembly" />
    <add mimeType="application/vnd.openxmlformats-officedocument.spreadsheetml.template"
         extractorType="Sitefinity.TextExtractors.XlsxTextExtractor, YourAssembly" />

    <!-- Macro-enabled Word only. Plain .docx
         (application/vnd.openxmlformats-officedocument.wordprocessingml.document) already has a
         working built-in extractor: leave it alone. -->
    <add mimeType="application/vnd.ms-word.document.macroEnabled.12"
         extractorType="Sitefinity.TextExtractors.WordTextExtractor, YourAssembly" />

    <!--
      NOT registered, on purpose: the legacy binary formats.

        .ppt  application/vnd.ms-powerpoint
        .doc  application/msword
        .xls  application/vnd.ms-excel

      These are OLE2 compound files, not ZIP packages, so the OpenXML SDK cannot read them and
      pointing these extractors at them only produces errors. If you need them indexed, NPOI
      reads all three and would need its own ITextExtractor implementation.
    -->

  </extractorSettings>
</documentServiceConfig>

using System;
using System.Collections.Generic;
using System.IO;
using System.Text;

namespace Sitefinity.TextExtractors
{
    internal static class TextExtractorOutput
    {
        /// <summary>
        /// Diagnostic context for a failed extraction.
        ///
        /// signature vs mimeType is the whole diagnosis when a document will not open, because
        /// Sitefinity derives the mime from the file extension and never from the bytes. A
        /// document declared as xlsx whose signature reads "ole2" is a password-protected or
        /// legacy binary file, and no amount of code will make the OpenXML SDK read it.
        ///
        /// streamLength is the other half: compare it against the size your storage claims for
        /// the document. Equal means the stored file really is damaged. Smaller means something
        /// in your read path is truncating, which is a genuine bug worth chasing.
        /// </summary>
        internal static Dictionary<string, string> BuildContext(string mimeType, Stream doc)
        {
            var context = new Dictionary<string, string>
            {
                { "mimeType", mimeType ?? "null" },
                { "signature", FileSignature.Describe(doc) }
            };

            // Length throws on some stream implementations; a partial context beats losing it all
            try
            {
                context["streamLength"] = doc?.Length.ToString() ?? "null";
            }
            catch (Exception ex)
            {
                context["streamLength"] = ex.GetType().Name;
            }

            // The extractor only ever sees a stream, never the document it came from. If you
            // want the title and id in your reports, stash them in an AsyncLocal from your
            // inbound pipe before extraction and merge them in here. Without that you are
            // correlating error reports to documents by timestamp, which is miserable.
            return context;
        }

        internal static void WriteUtf8(StringBuilder builder, Stream text)
        {
            // Raw byte write rather than a StreamWriter: disposing a writer would close the
            // caller's output stream before Sitefinity's DocumentService reads it back, and the
            // resulting "cannot access a closed stream" is a confusing way to learn that.
            var bytes = Encoding.UTF8.GetBytes(builder.ToString());
            text.Write(bytes, 0, bytes.Length);
        }
    }
}

using System;
using System.Collections.Specialized;
using System.IO;
using System.Text;
using Telerik.Sitefinity.Services.Documents;
using Telerik.Windows.Documents.Fixed.FormatProviders;
using Telerik.Windows.Documents.Fixed.FormatProviders.Pdf;
using Telerik.Windows.Documents.Fixed.FormatProviders.Text;
using Telerik.Windows.Documents.Fixed.Model;

namespace Sitefinity.TextExtractors
{
    /// <summary>
    /// Replaces Sitefinity's DefaultPdfTextExtractor.
    ///
    /// THE BUG IN THE STOCK ONE: it attaches its exception-tolerant handler to the document
    /// AFTER Import() returns, which is too late for the failures that matter. Structure-tree
    /// problems (InvalidStructureTreeException) and RichMedia annotations throw DURING import,
    /// so the handler is never reached and the whole document indexes with no body text. On an
    /// older library that is hundreds of files.
    ///
    /// Two changes fix it:
    ///   - subscribe DocumentUnhandledException on ImportSettings, so it is live during import
    ///   - set IgnoreMarkedContent, which skips the structure tree altogether
    ///
    /// The structure tree is accessibility and reading-order metadata. Text extraction never
    /// needs it, so skipping it costs nothing and removes an entire class of failure.
    ///
    /// Uses Telerik Document Processing, which already ships with Sitefinity.
    /// </summary>
    public class PdfTextExtractor : ITextExtractor
    {
        public string MimeType { get; private set; }

        public void Initialize(string mimeType, NameValueCollection config)
        {
            this.MimeType = mimeType;
        }

        public void GetText(Stream doc, Stream text)
        {
            ExtractorGuard.Run(nameof(PdfTextExtractor), "ExtractText", () => TextExtractorOutput.BuildContext(this.MimeType, doc), () =>
            {
                this.Extract(doc, text);
            });
        }

        private void Extract(Stream doc, Stream text)
        {
            var provider = new PdfFormatProvider();

            // OnDemand parses pages lazily instead of materialising the whole document up front,
            // which matters when a reindex walks thousands of files
            provider.ImportSettings.ReadingMode = ReadingMode.OnDemand;
            provider.ImportSettings.IgnoreMarkedContent = true;

            // The line the stock extractor gets wrong. Subscribing HERE, on the settings rather
            // than the document, is what makes it active while Import is running.
            provider.ImportSettings.DocumentUnhandledException += (sender, e) =>
            {
                e.Handled = true;
            };

            // Mirrors the stock extractor's use of the documentServiceConfig timeout (minutes)
            var timeout = TimeSpan.FromMinutes(5);

            RadFixedDocument document = provider.Import(doc, timeout);
            if (document == null)
            {
                return;
            }

            // And again on the document, for anything thrown during export rather than import
            document.DocumentUnhandledException += (sender, e) =>
            {
                e.Handled = true;
            };

            var exporter = new TextFormatProvider();
            var settings = new TextFormatProviderSettings("\r\n", string.Empty);
            var extracted = exporter.Export(document, settings, timeout);

            // A scanned PDF is a picture of text with no text layer, so extraction "succeeds"
            // with an empty string and the document indexes title-only with no error anywhere.
            // This is the seam where OCR would go. The branch is live but the call is not, so
            // measuring how many of your documents are scans is a one-line change.
            var looksLikeAScan = document.Pages.Count > 0
                && extracted.Trim().Length < document.Pages.Count * 20;
            if (looksLikeAScan)
            {
                // extracted = OcrPages(document);
            }

            TextExtractorOutput.WriteUtf8(new StringBuilder(extracted), text);
        }

        // NOT WIRED UP. Sketch for adding OCR against an external service (Azure AI Document
        // Intelligence, AWS Textract, a Tesseract sidecar; all take an image and return text).
        // Read these three before enabling it:
        //
        //   1. Billed per page, and a full reindex reprocesses every document. Cache results
        //      keyed by document id or every rebuild costs real money.
        //   2. Seconds per page, on the indexing thread. A few hundred scans turns a reindex
        //      from minutes into hours. If this becomes real, the right shape is a background
        //      job writing into an indexed field, not inline extraction.
        //   3. OCR returns confidently wrong words on bad scans, and those become real search
        //      terms. Usually still better than an empty document, but it is not the same
        //      quality bar as a genuine text layer, and users cannot tell the difference.
        //
        // private string OcrPages(RadFixedDocument document)
        // {
        //     var builder = new StringBuilder();
        //     foreach (var page in document.Pages)
        //     {
        //         // Telerik can rasterise a page for you; the provider type has moved between
        //         // versions (Skia-based currently), so check what your assemblies ship.
        //         byte[] pageImage;
        //         using (var buffer = new MemoryStream())
        //         {
        //             // imageProvider.Export(page, buffer);
        //             pageImage = buffer.ToArray();
        //         }
        //
        //         // Keep the per-page timeout short; one stalled call holds the whole reindex.
        //         // builder.AppendLine(ocrClient.Recognize(pageImage));
        //     }
        //     return builder.ToString();
        // }
    }
}

using System.Collections.Specialized;
using System.IO;
using System.Text;
using DocumentFormat.OpenXml.Packaging;
using Telerik.Sitefinity.Services.Documents;

namespace Sitefinity.TextExtractors
{
    /// <summary>
    /// Sitefinity ships no PowerPoint extractor at all, so pptx / pptm / ppsx documents index
    /// with a title and no body text. Uses the OpenXML SDK that already ships with Sitefinity,
    /// so there is no new dependency.
    ///
    /// Register one entry per mime type in DocumentServiceConfig.config; see registration.config.
    /// </summary>
    public class PptxTextExtractor : ITextExtractor
    {
        public string MimeType { get; private set; }

        public void Initialize(string mimeType, NameValueCollection config)
        {
            // Sitefinity constructs one instance per registered mime type and tells it which
            // one it is. PresentationDocument opens all three formats identically.
            this.MimeType = mimeType;
        }

        public void GetText(Stream doc, Stream text)
        {
            // A failure costs this document's body text only, never the indexed item
            ExtractorGuard.Run(nameof(PptxTextExtractor), "ExtractText", () => TextExtractorOutput.BuildContext(this.MimeType, doc), () =>
            {
                OpenXmlPackageReader.Read(doc, package => this.ExtractCore(package, text));
            });
        }

        /// <summary>
        /// Called by OpenXmlPackageReader, possibly twice: once on the original stream and, if
        /// the SDK refuses that, once on a repaired copy. Must therefore be safe to run twice.
        /// </summary>
        private void ExtractCore(Stream doc, Stream text)
        {
            var builder = new StringBuilder();

            // Deliberately not a using block. The SDK's close-time cleanup throws on a package
            // opened read-only, and that would discard the text collected just below it, so
            // disposal goes through DisposeQuietly instead.
            var presentation = PresentationDocument.Open(doc, false);
            try
            {
                var presentationPart = presentation.PresentationPart;
                if (presentationPart == null)
                {
                    return;
                }

                foreach (var slidePart in presentationPart.SlideParts)
                {
                    // Every piece of visible slide text is an a:t element, no matter how deeply
                    // it is nested in shapes, groups, tables or text boxes. Walking descendants
                    // gets all of it without modelling the shape tree at all.
                    foreach (var textNode in slidePart.Slide.Descendants<DocumentFormat.OpenXml.Drawing.Text>())
                    {
                        builder.AppendLine(textNode.Text);
                    }

                    // Speaker notes are often the most searchable text in the whole deck: the
                    // slide says "Management" over a diagram while the notes pane holds the
                    // actual prose somebody will search for months later.
                    var notes = slidePart.NotesSlidePart;
                    if (notes != null)
                    {
                        foreach (var noteText in notes.NotesSlide.Descendants<DocumentFormat.OpenXml.Drawing.Text>())
                        {
                            builder.AppendLine(noteText.Text);
                        }
                    }
                }
            }
            finally
            {
                OpenXmlPackageReader.DisposeQuietly(presentation);
            }

            TextExtractorOutput.WriteUtf8(builder, text);
        }
    }
}

using System;
using System.IO;
using DocumentFormat.OpenXml.Packaging;

namespace Sitefinity.TextExtractors
{
    /// <summary>
    /// Shared open policy for the three OOXML extractors (pptx, xlsx, docm).
    ///
    /// Every workaround for the OpenXML SDK lives here rather than in the extractors, because
    /// each extractor originally carried its own copy and the same gap then had to be found
    /// three separate times in production.
    ///
    /// The sequence is: reject input that is not a zip at all, try the straightforward open,
    /// and on the two known SDK failures retry against a repaired copy of the package.
    /// </summary>
    internal static class OpenXmlPackageReader
    {
        internal static void Read(Stream doc, Action<Stream> readPackage)
        {
            // Not a zip, so there is no OOXML package inside it. See the FileSignature header
            // for why a non-zip reaches an OOXML extractor in the first place.
            if (!FileSignature.IsZip(doc))
            {
                return;
            }

            try
            {
                readPackage(doc);
            }
            catch (FileFormatException)
            {
                // Starts with the zip magic but has no central directory, so the upload is
                // truncated or damaged. Unlike the SDK defects below there is nothing to strip
                // and retry: a zip with no directory cannot be read by anything. Skip it the
                // same way a non-zip is skipped, and let the document index on its title.
                //
                // This is a deliberate trade. Skipping silently means you can no longer see
                // WHICH files are damaged. If you would rather find and re-upload them, delete
                // this catch and let the guard report them (capped) instead.
                return;
            }
            catch (Exception ex) when (IsRecoverableOpenFailure(ex))
            {
                // The SDK refused the package. The known cause is printer-settings parts, so
                // retry against a copy with those removed.
                Stream sanitized = null;
                try
                {
                    sanitized = OpenXmlPackageSanitizer.StripPrinterSettings(doc);
                }
                catch
                {
                    // A package broken some other way makes the sanitizer throw on its own.
                    // Letting that escape would report the failed recovery instead of the
                    // actual fault, which is a much harder thing to diagnose later.
                }

                if (sanitized == null)
                {
                    throw;
                }

                using (sanitized)
                {
                    readPackage(sanitized);
                }
            }
        }

        /// <summary>
        /// Both exception types mean the same thing here: the SDK could not load this package,
        /// and a printerSettings-stripped copy is worth trying.
        /// </summary>
        private static bool IsRecoverableOpenFailure(Exception ex)
        {
            if (ex is OpenXmlPackageException)
            {
                return true;
            }

            // IOException looks unrelated and is not. When Load() fails, its own cleanup path
            // calls Close() -> DeleteUnusedDataPartOnClose() -> Package.DeletePart(), and that
            // throws IOException("Cannot modify a read-only container") because the package was
            // opened read-only. The cleanup failure REPLACES the OpenXmlPackageException that
            // caused it, so matching only on the latter means the retry below never runs and
            // you are left staring at a read-only error that explains nothing.
            return ex is IOException;
        }

        /// <summary>
        /// Disposes a package opened read-only, tolerating the same SDK cleanup defect.
        ///
        /// DeleteUnusedDataPartOnClose runs on EVERY dispose, not only failed loads, so a
        /// document that opened and extracted perfectly can still throw on the closing brace of
        /// a using block. Callers write their extracted text after disposing, so letting that
        /// through would silently discard work that already succeeded.
        /// </summary>
        internal static void DisposeQuietly(OpenXmlPackage package)
        {
            if (package == null)
            {
                return;
            }

            try
            {
                package.Dispose();
            }
            catch (IOException)
            {
                // Safe precisely because the package is read-only: there are no pending writes
                // to lose. The same swallow on a writable package would be a real bug.
            }
        }
    }
}

using System;
using System.IO;
using System.Linq;

namespace Sitefinity.TextExtractors
{
    /// <summary>
    /// Works around a defect in OpenXML SDK 2.0 (2.0.5022.0, the build Sitefinity ships).
    ///
    /// The SDK's content-type expectation table maps EVERY printerSettings part to the
    /// SPREADSHEET printer-settings content type. So a pptx or docm saved on a machine with a
    /// printer configured, which is most of them, fails to open with:
    ///
    ///     The document cannot be opened because there is an invalid part with an unexpected
    ///     content type. [Part Uri=/ppt/printerSettings/printerSettings1.bin] ...
    ///     [Expected Content Type=...spreadsheetml.printerSettings]
    ///
    /// The part is fine. The SDK's expectation is wrong. It is fixed in SDK 2.5+, but you
    /// generally cannot upgrade that assembly out from under Sitefinity's own dependency.
    ///
    /// Printer settings carry no indexable text, so the recovery is to hand back a copy of the
    /// package with those parts removed and let the caller retry.
    /// </summary>
    internal static class OpenXmlPackageSanitizer
    {
        /// <summary>
        /// Returns a seekable in-memory copy of the package with all printerSettings parts and
        /// the relationships pointing at them removed. Returns null when there is nothing to
        /// strip or the source cannot be re-read, in which case the caller should rethrow the
        /// original open failure rather than pretend it recovered.
        /// </summary>
        internal static Stream StripPrinterSettings(Stream doc)
        {
            // The failed open already consumed part of the stream. Without seek there is
            // nothing left to copy, so the caller keeps its original exception.
            if (doc == null || !doc.CanSeek)
            {
                return null;
            }
            doc.Seek(0, SeekOrigin.Begin);

            var working = new MemoryStream();
            doc.CopyTo(working);
            working.Seek(0, SeekOrigin.Begin);

            // System.IO.Packaging lives in WindowsBase and is fully qualified throughout this
            // file, because DocumentFormat.OpenXml.Packaging has colliding type names.
            using (var package = System.IO.Packaging.Package.Open(working, FileMode.Open, FileAccess.ReadWrite))
            {
                var printerParts = package.GetParts()
                    .Where(part => part.Uri.OriginalString.IndexOf("printerSettings", StringComparison.OrdinalIgnoreCase) >= 0)
                    .Select(part => part.Uri)
                    .ToList();

                if (printerParts.Count == 0)
                {
                    return null;
                }

                // A relationship pointing at a part that no longer exists fails validation just
                // like the bad part did, so the relationships go first.
                foreach (var part in package.GetParts().ToList())
                {
                    // Relationship parts cannot themselves carry relationships; asking throws.
                    if (part.Uri.OriginalString.EndsWith(".rels", StringComparison.OrdinalIgnoreCase))
                    {
                        continue;
                    }
                    foreach (var rel in part.GetRelationships().ToList())
                    {
                        if (!IsInternalTarget(rel))
                        {
                            continue;
                        }
                        var target = System.IO.Packaging.PackUriHelper.ResolvePartUri(part.Uri, rel.TargetUri);
                        if (printerParts.Contains(target))
                        {
                            part.DeleteRelationship(rel.Id);
                        }
                    }
                }

                foreach (var rel in package.GetRelationships().ToList())
                {
                    if (!IsInternalTarget(rel))
                    {
                        continue;
                    }
                    // PackageRootUri is a .NET Core addition; on Framework, resolve against "/".
                    var target = System.IO.Packaging.PackUriHelper.ResolvePartUri(new Uri("/", UriKind.Relative), rel.TargetUri);
                    if (printerParts.Contains(target))
                    {
                        package.DeleteRelationship(rel.Id);
                    }
                }

                foreach (var uri in printerParts)
                {
                    package.DeletePart(uri);
                }
            }

            // Package.Close disposes the working stream. MemoryStream.ToArray still reads after
            // dispose, so hand back a fresh stream over the finished bytes.
            return new MemoryStream(working.ToArray());
        }

        /// <summary>
        /// The single most important line in this file.
        ///
        /// An EXTERNAL relationship, which is what an ordinary hyperlink is, carries an absolute
        /// URI, and PackUriHelper.ResolvePartUri throws ArgumentException("Cannot be an absolute
        /// URI") on those. Without this guard the sanitizer throws on any document containing a
        /// link, the caller treats the package as unrecoverable, and the original error is
        /// rethrown as though no recovery had been attempted.
        ///
        /// This is easy to miss because a hand-built test package has no hyperlinks and passes.
        /// Real documents nearly always have them.
        /// </summary>
        private static bool IsInternalTarget(System.IO.Packaging.PackageRelationship relationship)
        {
            return relationship.TargetMode == System.IO.Packaging.TargetMode.Internal
                && !relationship.TargetUri.IsAbsoluteUri;
        }
    }
}

using System;
using System.IO;

namespace Sitefinity.TextExtractors
{
    /// <summary>
    /// Magic-number sniffing for the document text extractors.
    ///
    /// Sitefinity picks an extractor from the document's stored mime type, which is derived from
    /// the file EXTENSION at upload time and never from the bytes. Two consequences show up in
    /// any real library:
    ///
    ///   - a legacy binary .xls renamed to .xlsx reaches the OOXML extractor and is not a zip
    ///   - far more commonly, a password-protected Office file is not a zip either, because
    ///     encryption wraps the entire OOXML package inside an OLE2 compound file
    ///
    /// The OpenXML SDK can never read either, and its failure (FileFormatException, "File
    /// contains corrupted data") is a property of the upload rather than a defect to fix. So the
    /// extractors check the signature first and skip, instead of reporting an error nobody can
    /// action.
    /// </summary>
    internal static class FileSignature
    {
        /// <summary>
        /// True when the stream starts with the zip magic every OOXML package must begin with.
        /// A stream that cannot be sniffed gets the benefit of the doubt: better to let the SDK
        /// try and fail than to skip a file that was perfectly readable.
        /// </summary>
        internal static bool IsZip(Stream doc)
        {
            var header = ReadHeader(doc);
            if (header == null)
            {
                return true;
            }

            return IsZip(header);
        }

        /// <summary>
        /// Short label for error-report context, so a future failure arrives already diagnosed
        /// instead of as a bare "corrupted data". Comparing this against the document's declared
        /// mime type is usually the whole diagnosis. Never throws.
        /// </summary>
        internal static string Describe(Stream doc)
        {
            try
            {
                var header = ReadHeader(doc);
                if (header == null)
                {
                    return "unreadable";
                }
                if (header.Length == 0)
                {
                    return "empty";
                }
                if (IsZip(header))
                {
                    return "zip";
                }
                if (StartsWith(header, 0xD0, 0xCF, 0x11, 0xE0))
                {
                    return "ole2 (legacy or password-protected office file)";
                }
                if (StartsWith(header, 0x25, 0x50, 0x44, 0x46))
                {
                    return "pdf";
                }

                return BitConverter.ToString(header);
            }
            catch (Exception ex)
            {
                return ex.GetType().Name;
            }
        }

        // "PK" covers the local-header, empty-archive and spanned-archive variants
        private static bool IsZip(byte[] header)
        {
            return StartsWith(header, 0x50, 0x4B);
        }

        private static bool StartsWith(byte[] header, params byte[] magic)
        {
            if (header.Length < magic.Length)
            {
                return false;
            }

            for (var i = 0; i < magic.Length; i++)
            {
                if (header[i] != magic[i])
                {
                    return false;
                }
            }

            return true;
        }

        /// <summary>
        /// Returns the first bytes of the package, or null when the stream cannot be sniffed
        /// without consuming bytes the caller still needs. Always restores the original
        /// position, because the SDK reads the same stream immediately afterwards.
        /// </summary>
        private static byte[] ReadHeader(Stream doc)
        {
            if (doc == null || !doc.CanRead || !doc.CanSeek)
            {
                return null;
            }

            var origin = doc.Position;
            try
            {
                doc.Seek(0, SeekOrigin.Begin);

                // Read can return short of the request even with bytes remaining, so loop
                var buffer = new byte[8];
                var filled = 0;
                while (filled < buffer.Length)
                {
                    var read = doc.Read(buffer, filled, buffer.Length - filled);
                    if (read <= 0)
                    {
                        break;
                    }
                    filled += read;
                }

                var header = new byte[filled];
                Array.Copy(buffer, header, filled);
                return header;
            }
            finally
            {
                doc.Seek(origin, SeekOrigin.Begin);
            }
        }
    }
}

using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Text;
using DocumentFormat.OpenXml.Packaging;
using Telerik.Sitefinity.Services.Documents;

namespace Sitefinity.TextExtractors
{
    /// <summary>
    /// Sitefinity ships no spreadsheet extractor, so xlsx / xlsm / xltx documents index with a
    /// title and no body text.
    ///
    /// Register one entry per mime type in DocumentServiceConfig.config; see registration.config.
    /// </summary>
    public class XlsxTextExtractor : ITextExtractor
    {
        public string MimeType { get; private set; }

        public void Initialize(string mimeType, NameValueCollection config)
        {
            // SpreadsheetDocument opens xlsm and xltx the same way it opens xlsx
            this.MimeType = mimeType;
        }

        public void GetText(Stream doc, Stream text)
        {
            ExtractorGuard.Run(nameof(XlsxTextExtractor), "ExtractText", () => TextExtractorOutput.BuildContext(this.MimeType, doc), () =>
            {
                OpenXmlPackageReader.Read(doc, package => this.ExtractCore(package, text));
            });
        }

        private void ExtractCore(Stream doc, Stream text)
        {
            var builder = new StringBuilder();

            // Not a using block: see DisposeQuietly in OpenXmlPackageReader
            var spreadsheet = SpreadsheetDocument.Open(doc, false);
            try
            {
                var workbookPart = spreadsheet.WorkbookPart;
                if (workbookPart == null)
                {
                    return;
                }

                // THE THING THAT SURPRISES PEOPLE ABOUT XLSX: cell text is not stored in the
                // cell. Strings live once in a shared-string table and each cell holds an
                // integer index into it, so a naive walk over cells hands you a pile of numbers
                // and no words. Load the table first, then dereference.
                var sharedStrings = new List<string>();
                var sharedStringPart = workbookPart.SharedStringTablePart;
                if (sharedStringPart != null)
                {
                    sharedStrings = sharedStringPart.SharedStringTable
                        .Elements<DocumentFormat.OpenXml.Spreadsheet.SharedStringItem>()
                        .Select(item => item.InnerText)
                        .ToList();
                }

                foreach (var sheetPart in workbookPart.WorksheetParts)
                {
                    foreach (var cell in sheetPart.Worksheet.Descendants<DocumentFormat.OpenXml.Spreadsheet.Cell>())
                    {
                        if (cell.CellValue == null)
                        {
                            continue;
                        }

                        var value = cell.CellValue.InnerText;
                        if (cell.DataType != null && cell.DataType.Value == DocumentFormat.OpenXml.Spreadsheet.CellValues.SharedString)
                        {
                            // Bounds-check rather than trust the index: a corrupt or truncated
                            // shared-string table would otherwise throw for the whole document
                            // when the cost of one bad cell should be one bad cell.
                            int index;
                            if (int.TryParse(value, out index) && index >= 0 && index < sharedStrings.Count)
                            {
                                builder.AppendLine(sharedStrings[index]);
                            }
                        }
                        else
                        {
                            // Inline strings, numbers and dates. Numbers are worth keeping:
                            // people search for order numbers and student ids.
                            builder.AppendLine(value);
                        }
                    }
                }
            }
            finally
            {
                OpenXmlPackageReader.DisposeQuietly(spreadsheet);
            }

            TextExtractorOutput.WriteUtf8(builder, text);
        }
    }
}

using System.Collections.Specialized;
using System.IO;
using System.Text;
using DocumentFormat.OpenXml.Packaging;
using Telerik.Sitefinity.Services.Documents;

namespace Sitefinity.TextExtractors
{
    /// <summary>
    /// Covers macro-enabled Word (.docm).
    ///
    /// Sitefinity's built-in extractor is registered against the docx mime type only, and docm
    /// has a different one, so macro-enabled documents fall through to no extractor at all and
    /// index title-only. The file format is otherwise identical, which is why this is short.
    ///
    /// Plain .docx should stay with the built-in DefaultTextExtractor: do not register this
    /// against that mime type, there is nothing to gain.
    /// </summary>
    public class WordTextExtractor : ITextExtractor
    {
        public string MimeType { get; private set; }

        public void Initialize(string mimeType, NameValueCollection config)
        {
            this.MimeType = mimeType;
        }

        public void GetText(Stream doc, Stream text)
        {
            ExtractorGuard.Run(nameof(WordTextExtractor), "ExtractText", () => TextExtractorOutput.BuildContext(this.MimeType, doc), () =>
            {
                OpenXmlPackageReader.Read(doc, package => this.ExtractCore(package, text));
            });
        }

        private void ExtractCore(Stream doc, Stream text)
        {
            var builder = new StringBuilder();

            // Not a using block: see DisposeQuietly in OpenXmlPackageReader
            var word = WordprocessingDocument.Open(doc, false);
            try
            {
                var body = word.MainDocumentPart?.Document?.Body;
                if (body == null)
                {
                    return;
                }

                // Paragraph.InnerText concatenates every run inside the paragraph, which is what
                // you want: Word splits a single sentence across runs whenever formatting or
                // spell-check state changes mid-line, so reading runs individually would shred
                // words into fragments that match nothing.
                foreach (var paragraph in body.Descendants<DocumentFormat.OpenXml.Wordprocessing.Paragraph>())
                {
                    builder.AppendLine(paragraph.InnerText);
                }

                // Note: this reads the document body only. Headers, footers and footnotes live
                // in separate parts (word.MainDocumentPart.HeaderParts and friends) and are
                // usually boilerplate, so they are skipped on purpose.
            }
            finally
            {
                OpenXmlPackageReader.DisposeQuietly(word);
            }

            TextExtractorOutput.WriteUtf8(builder, text);
        }
    }
}

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;

namespace Sitefinity.TextExtractors
{
    /// <summary>
    /// Containment boundary for search-index extraction.
    ///
    /// WHY THIS EXISTS, because it looks like ceremony around a try/catch:
    /// Telerik runs every inbound pipe through PublishingHelper.ForEachSafe, which swallows
    /// exceptions and silently DROPS the item from the Lucene index. An unguarded
    /// NullReferenceException in a document pipe will quietly unindex documents for as long as
    /// it takes somebody to notice they cannot find a file. Wrapping each step caps the blast
    /// radius of a failure at one document's body text instead of the whole item, and reports it
    /// somewhere a human will actually see.
    ///
    /// THE CAP IS NOT OPTIONAL: a systemic data problem during a full reindex fails once per
    /// item, so 20,000 documents means 20,000 identical error reports, which is the same as
    /// having none. The first few carry the signal and the counter preserves the true scale.
    /// </summary>
    internal static class ExtractorGuard
    {
        /// <summary>
        /// Wire this up once at startup to Sentry, Raygun, log4net, or whatever you use:
        /// (exception, contextData, message). Left null, failures are contained silently.
        /// </summary>
        internal static Action<Exception, Dictionary<string, string>, string> Report;

        private const int MaxReportsPerStep = 10;
        private static readonly ConcurrentDictionary<string, int> ReportCounts = new ConcurrentDictionary<string, int>();

        internal static void Run(string pipeName, string step, Func<Dictionary<string, string>> contextBuilder, Action action)
        {
            try
            {
                action();
            }
            catch (Exception ex)
            {
                var failureCount = ReportCounts.AddOrUpdate($"{pipeName}.{step}", 1, (key, count) => count + 1);
                if (failureCount > MaxReportsPerStep)
                {
                    return;
                }

                var customData = new Dictionary<string, string>
                {
                    { "pipe", pipeName },
                    { "step", step },
                    { "failureCountThisAppDomain", failureCount.ToString() },
                    { "reportingCapped", (failureCount == MaxReportsPerStep).ToString() }
                };

                // Context readers touch lazy-loaded Sitefinity properties and may throw on a
                // background thread. A catch block that throws is worse than the original bug,
                // so a failure to build context must never mask the exception being reported.
                try
                {
                    if (contextBuilder != null)
                    {
                        foreach (var pair in contextBuilder())
                        {
                            customData[pair.Key] = pair.Value ?? "null";
                        }
                    }
                }
                catch (Exception contextEx)
                {
                    customData["contextError"] = contextEx.Message;
                }

                var report = Report;
                if (report != null)
                {
                    report(ex, customData, $"{pipeName}.{step} failed; the item was indexed without this data");
                }
            }
        }
    }
}