Avoid stack overflow when loading CPDF_Function.

CPDF_StitchFuncs that reference each other create a Load() loop.
Maintaining a set of the visited CPDF_Objects during a Load()
call tree prevents that.

Bug: chromium:830221
Change-Id: I6f494da16c6d79f05870ff85cff38ff8fe69ecfe
Reviewed-on: https://pdfium-review.googlesource.com/30050
Commit-Queue: dsinclair <dsinclair@chromium.org>
Reviewed-by: dsinclair <dsinclair@chromium.org>
diff --git a/core/fpdfapi/page/cpdf_expintfunc.cpp b/core/fpdfapi/page/cpdf_expintfunc.cpp
index 5d12131..f26df1b 100644
--- a/core/fpdfapi/page/cpdf_expintfunc.cpp
+++ b/core/fpdfapi/page/cpdf_expintfunc.cpp
@@ -20,7 +20,8 @@
   FX_Free(m_pEndValues);
 }
 
-bool CPDF_ExpIntFunc::v_Init(CPDF_Object* pObj) {
+bool CPDF_ExpIntFunc::v_Init(CPDF_Object* pObj,
+                             std::set<CPDF_Object*>* pVisited) {
   CPDF_Dictionary* pDict = pObj->GetDict();
   if (!pDict)
     return false;
diff --git a/core/fpdfapi/page/cpdf_expintfunc.h b/core/fpdfapi/page/cpdf_expintfunc.h
index 867d2fa..f46d6c2 100644
--- a/core/fpdfapi/page/cpdf_expintfunc.h
+++ b/core/fpdfapi/page/cpdf_expintfunc.h
@@ -7,6 +7,8 @@
 #ifndef CORE_FPDFAPI_PAGE_CPDF_EXPINTFUNC_H_
 #define CORE_FPDFAPI_PAGE_CPDF_EXPINTFUNC_H_
 
+#include <set>
+
 #include "core/fpdfapi/page/cpdf_function.h"
 
 class CPDF_ExpIntFunc : public CPDF_Function {
@@ -15,7 +17,7 @@
   ~CPDF_ExpIntFunc() override;
 
   // CPDF_Function
-  bool v_Init(CPDF_Object* pObj) override;
+  bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) override;
   bool v_Call(float* inputs, float* results) const override;
 
   uint32_t m_nOrigOutputs;
diff --git a/core/fpdfapi/page/cpdf_function.cpp b/core/fpdfapi/page/cpdf_function.cpp
index 635c53a..3e99eb8 100644
--- a/core/fpdfapi/page/cpdf_function.cpp
+++ b/core/fpdfapi/page/cpdf_function.cpp
@@ -17,10 +17,22 @@
 
 // static
 std::unique_ptr<CPDF_Function> CPDF_Function::Load(CPDF_Object* pFuncObj) {
+  std::set<CPDF_Object*> visited;
+  return Load(pFuncObj, &visited);
+}
+
+// static
+std::unique_ptr<CPDF_Function> CPDF_Function::Load(
+    CPDF_Object* pFuncObj,
+    std::set<CPDF_Object*>* pVisited) {
   std::unique_ptr<CPDF_Function> pFunc;
   if (!pFuncObj)
     return pFunc;
 
+  if (pdfium::ContainsKey(*pVisited, pFuncObj))
+    return nullptr;
+  pdfium::ScopedSetInsertion<CPDF_Object*> insertion(pVisited, pFuncObj);
+
   int iType = -1;
   if (CPDF_Stream* pStream = pFuncObj->AsStream())
     iType = pStream->GetDict()->GetIntegerFor("FunctionType");
@@ -37,7 +49,7 @@
   else if (type == Type::kType4PostScript)
     pFunc = pdfium::MakeUnique<CPDF_PSFunc>();
 
-  if (!pFunc || !pFunc->Init(pFuncObj))
+  if (!pFunc || !pFunc->Init(pFuncObj, pVisited))
     return nullptr;
 
   return pFunc;
@@ -64,7 +76,7 @@
   FX_Free(m_pRanges);
 }
 
-bool CPDF_Function::Init(CPDF_Object* pObj) {
+bool CPDF_Function::Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) {
   CPDF_Stream* pStream = pObj->AsStream();
   CPDF_Dictionary* pDict = pStream ? pStream->GetDict() : pObj->AsDictionary();
 
@@ -89,7 +101,7 @@
       m_pRanges[i] = pRanges->GetFloatAt(i);
   }
   uint32_t old_outputs = m_nOutputs;
-  if (!v_Init(pObj))
+  if (!v_Init(pObj, pVisited))
     return false;
   if (m_pRanges && m_nOutputs > old_outputs) {
     m_pRanges = FX_Realloc(float, m_pRanges, m_nOutputs * 2);
diff --git a/core/fpdfapi/page/cpdf_function.h b/core/fpdfapi/page/cpdf_function.h
index ff7cf84..7966194 100644
--- a/core/fpdfapi/page/cpdf_function.h
+++ b/core/fpdfapi/page/cpdf_function.h
@@ -8,6 +8,7 @@
 #define CORE_FPDFAPI_PAGE_CPDF_FUNCTION_H_
 
 #include <memory>
+#include <set>
 
 class CPDF_ExpIntFunc;
 class CPDF_Object;
@@ -50,8 +51,10 @@
  protected:
   explicit CPDF_Function(Type type);
 
-  bool Init(CPDF_Object* pObj);
-  virtual bool v_Init(CPDF_Object* pObj) = 0;
+  static std::unique_ptr<CPDF_Function> Load(CPDF_Object* pFuncObj,
+                                             std::set<CPDF_Object*>* pVisited);
+  bool Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited);
+  virtual bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) = 0;
   virtual bool v_Call(float* inputs, float* results) const = 0;
 
   uint32_t m_nInputs;
diff --git a/core/fpdfapi/page/cpdf_psfunc.cpp b/core/fpdfapi/page/cpdf_psfunc.cpp
index 5499a47..6df7dff 100644
--- a/core/fpdfapi/page/cpdf_psfunc.cpp
+++ b/core/fpdfapi/page/cpdf_psfunc.cpp
@@ -13,7 +13,7 @@
 
 CPDF_PSFunc::~CPDF_PSFunc() {}
 
-bool CPDF_PSFunc::v_Init(CPDF_Object* pObj) {
+bool CPDF_PSFunc::v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) {
   auto pAcc = pdfium::MakeRetain<CPDF_StreamAcc>(pObj->AsStream());
   pAcc->LoadAllDataFiltered();
   return m_PS.Parse(pAcc->GetSpan());
diff --git a/core/fpdfapi/page/cpdf_psfunc.h b/core/fpdfapi/page/cpdf_psfunc.h
index b8c18c9..7212fea 100644
--- a/core/fpdfapi/page/cpdf_psfunc.h
+++ b/core/fpdfapi/page/cpdf_psfunc.h
@@ -7,6 +7,8 @@
 #ifndef CORE_FPDFAPI_PAGE_CPDF_PSFUNC_H_
 #define CORE_FPDFAPI_PAGE_CPDF_PSFUNC_H_
 
+#include <set>
+
 #include "core/fpdfapi/page/cpdf_function.h"
 #include "core/fpdfapi/page/cpdf_psengine.h"
 
@@ -18,7 +20,7 @@
   ~CPDF_PSFunc() override;
 
   // CPDF_Function
-  bool v_Init(CPDF_Object* pObj) override;
+  bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) override;
   bool v_Call(float* inputs, float* results) const override;
 
  private:
diff --git a/core/fpdfapi/page/cpdf_sampledfunc.cpp b/core/fpdfapi/page/cpdf_sampledfunc.cpp
index 98e1fd0..78c4f2c 100644
--- a/core/fpdfapi/page/cpdf_sampledfunc.cpp
+++ b/core/fpdfapi/page/cpdf_sampledfunc.cpp
@@ -36,7 +36,8 @@
 
 CPDF_SampledFunc::~CPDF_SampledFunc() {}
 
-bool CPDF_SampledFunc::v_Init(CPDF_Object* pObj) {
+bool CPDF_SampledFunc::v_Init(CPDF_Object* pObj,
+                              std::set<CPDF_Object*>* pVisited) {
   CPDF_Stream* pStream = pObj->AsStream();
   if (!pStream)
     return false;
diff --git a/core/fpdfapi/page/cpdf_sampledfunc.h b/core/fpdfapi/page/cpdf_sampledfunc.h
index d6dfed5..b06b14b 100644
--- a/core/fpdfapi/page/cpdf_sampledfunc.h
+++ b/core/fpdfapi/page/cpdf_sampledfunc.h
@@ -7,6 +7,7 @@
 #ifndef CORE_FPDFAPI_PAGE_CPDF_SAMPLEDFUNC_H_
 #define CORE_FPDFAPI_PAGE_CPDF_SAMPLEDFUNC_H_
 
+#include <set>
 #include <vector>
 
 #include "core/fpdfapi/page/cpdf_function.h"
@@ -30,7 +31,7 @@
   ~CPDF_SampledFunc() override;
 
   // CPDF_Function
-  bool v_Init(CPDF_Object* pObj) override;
+  bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) override;
   bool v_Call(float* inputs, float* results) const override;
 
   const std::vector<SampleEncodeInfo>& GetEncodeInfo() const {
diff --git a/core/fpdfapi/page/cpdf_stitchfunc.cpp b/core/fpdfapi/page/cpdf_stitchfunc.cpp
index 336c74b..fd83fd6 100644
--- a/core/fpdfapi/page/cpdf_stitchfunc.cpp
+++ b/core/fpdfapi/page/cpdf_stitchfunc.cpp
@@ -22,7 +22,8 @@
   FX_Free(m_pEncode);
 }
 
-bool CPDF_StitchFunc::v_Init(CPDF_Object* pObj) {
+bool CPDF_StitchFunc::v_Init(CPDF_Object* pObj,
+                             std::set<CPDF_Object*>* pVisited) {
   CPDF_Dictionary* pDict = pObj->GetDict();
   if (!pDict) {
     return false;
@@ -42,7 +43,7 @@
     CPDF_Object* pSub = pArray->GetDirectObjectAt(i);
     if (pSub == pObj)
       return false;
-    std::unique_ptr<CPDF_Function> pFunc(CPDF_Function::Load(pSub));
+    std::unique_ptr<CPDF_Function> pFunc(CPDF_Function::Load(pSub, pVisited));
     if (!pFunc)
       return false;
     // Check that the input dimensionality is 1, and that all output
diff --git a/core/fpdfapi/page/cpdf_stitchfunc.h b/core/fpdfapi/page/cpdf_stitchfunc.h
index 69e5e41..11396f1 100644
--- a/core/fpdfapi/page/cpdf_stitchfunc.h
+++ b/core/fpdfapi/page/cpdf_stitchfunc.h
@@ -8,6 +8,7 @@
 #define CORE_FPDFAPI_PAGE_CPDF_STITCHFUNC_H_
 
 #include <memory>
+#include <set>
 #include <vector>
 
 #include "core/fpdfapi/page/cpdf_function.h"
@@ -18,7 +19,7 @@
   ~CPDF_StitchFunc() override;
 
   // CPDF_Function
-  bool v_Init(CPDF_Object* pObj) override;
+  bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) override;
   bool v_Call(float* inputs, float* results) const override;
 
   const std::vector<std::unique_ptr<CPDF_Function>>& GetSubFunctions() const {