URLArchive         Archive class for loading resources from an URL

I had need to load resources from a HTTP server recently, and wrote a class that plugs into the OgreArchive hierarchy and loads materials from a web server (or ftp server or any other kind of URL). The library I am using for this is libcurl, but it should be easy to plug your own code if you like. The following is a first shot - many improvements are possible.

Initialize libcurl

Somewhere in your main.cpp file, add the following lines:

#include <curl/curl.h>
...
curl_global_init(CURL_GLOBAL_DEFAULT);
... // app.go() and exception handling go here
curl_global_cleanup();


Also, make sure you add libcurl.lib to your linker settings.

URLArchive.cpp

#include "stdafx.h"

#include <OgreStableHeaders.h>
#include "URLArchive.h"
#include <OgreLogManager.h>
#include <OgreException.h>
#include <OgreConfigFile.h>
#include <OgreStringVector.h>
#include <OgreRoot.h>
#include <OgreDataStream.h>

#include <curl/curl.h>

using namespace Ogre;

//-----------------------------------------------------------------------
URLArchive::URLArchive(const String& name, const String& archType )
: Archive(name, archType), mCurl(0), mBuffer(0)
{
}
//-----------------------------------------------------------------------
URLArchive::~URLArchive()
{
  unload();
}
//-----------------------------------------------------------------------
struct MemoryStruct {
  unsigned  char * memory;
  size_t size;
};

static CURLcode 
fetch_url(CURL * curl, void *& buffer, size_t& size)
{
  MemoryStruct data = { 0 };
  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
  CURLcode res = curl_easy_perform(curl);
  buffer = data.memory;
  size = data.size;
  return res;
}

static size_t
write_memory_callback(void *ptr, size_t size, size_t nmemb, void *data)
{
  size_t realsize = size * nmemb;
  struct MemoryStruct *mem = (struct MemoryStruct *)data;

  assert(mem->memory!=NULL || mem->size==0);
  mem->memory = (unsigned char *)realloc(mem->memory, mem->size + realsize + 1);
  if (mem->memory) {
    memcpy(&(mem->memory[mem->size]), ptr, realsize);
    mem->size += realsize;
    mem->memory[mem->size] = 0;
  }
  return realsize;
}

void
URLArchive::load()
{
  mCurl = curl_easy_init();
  curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, write_memory_callback);
}
//-----------------------------------------------------------------------
void
URLArchive::unload()
{
  if (mCurl) {
    curl_easy_cleanup(mCurl);
    mCurl = 0;
  }
  if (mBuffer) {
    free(mBuffer);
    mBuffer = 0;
  }
}
//-----------------------------------------------------------------------
DataStreamPtr
URLArchive::open(const String& filename) const
{
  void * buffer;
  size_t size;
  std::string url = mName + filename;

  if (mBuffer==NULL || url!=mLastURL) {
    curl_easy_setopt(mCurl, CURLOPT_URL, url.c_str());
    CURLcode res = fetch_url(mCurl, buffer, size);

    long statLong = 0;
    if (CURLE_OK == res) {
      curl_easy_getinfo(mCurl, CURLINFO_HTTP_CODE, &statLong);
    }
    if (statLong!=200 || buffer==NULL) {
      OGRE_EXCEPT(Exception::ERR_FILE_NOT_FOUND,
        "Could not open resource: " + filename, "RDBArchive::open");
    }
  } else {
    // recycle the data we got in the call to exist()
    buffer = mBuffer;
    size = mSize;
    mBuffer = NULL;
  }

  MemoryDataStream stream(buffer, size, false);
  DataStreamPtr ptr(new MemoryDataStream(stream, true));
  free(buffer);
  return ptr;
}
//-----------------------------------------------------------------------
StringVectorPtr URLArchive::list(bool recursive, bool dirs)
{
  // directory change requires locking due to saved returns
  return StringVectorPtr(new StringVector());
}
//-----------------------------------------------------------------------
FileInfoListPtr URLArchive::listFileInfo(bool recursive, bool dirs)
{
  return FileInfoListPtr(new FileInfoList());
}
//-----------------------------------------------------------------------
StringVectorPtr URLArchive::find(const String& pattern,
                                 bool recursive, bool dirs)
{
  return StringVectorPtr(new StringVector());
}
//-----------------------------------------------------------------------
FileInfoListPtr URLArchive::findFileInfo(const String& pattern, 
                                         bool recursive, bool dirs)
{
  return FileInfoListPtr(new FileInfoList());
}
//-----------------------------------------------------------------------
bool
URLArchive::exists(const String& filename)
{
  std::string url = mName + filename;
  if (mBuffer) {
    free(mBuffer);
    mBuffer = 0;
  }

  curl_easy_setopt(mCurl, CURLOPT_URL, url.c_str());
  CURLcode res = fetch_url(mCurl, mBuffer, mSize);

  long statLong = 0;
  if (CURLE_OK == res) {
    curl_easy_getinfo(mCurl, CURLINFO_HTTP_CODE, &statLong);
  }
  if (statLong!=200) {
    if (mBuffer) free(mBuffer);
    mBuffer = NULL;
  }

  return (statLong==200);
}
//-----------------------------------------------------------------------
const String& URLArchiveFactory::getType(void) const
{
  static String name = "URL";
  return name;
}

URLArchive.h

#ifndef __URLArchive_H__
#define __URLArchive_H__

#include <OgrePrerequisites.h>

#include <OgreArchive.h>
#include <OgreArchiveFactory.h>
#include <OgreArchive.h>

/** Specialisation of the Archive class to allow reading of files from a URL.
*/
class URLArchive : public Ogre::Archive 
{
public:
  URLArchive(const Ogre::String& name, const Ogre::String& archType );
  ~URLArchive();

  /// @copydoc Archive::isCaseSensitive
  bool isCaseSensitive(void) const { return true; }

  /// @copydoc Archive::load
  void load();
  /// @copydoc Archive::unload
  void unload();

  /// @copydoc Archive::open
  Ogre::DataStreamPtr open(const Ogre::String& filename) const;

  /// @copydoc Archive::list
  Ogre::StringVectorPtr list(bool recursive = true, bool dirs = false);

  /// @copydoc Archive::listFileInfo
  Ogre::FileInfoListPtr listFileInfo(bool recursive = true, bool dirs = false);

  /// @copydoc Archive::find
  Ogre::StringVectorPtr find(const Ogre::String& pattern, bool recursive = true,
    bool dirs = false);

  /// @copydoc Archive::findFileInfo
  Ogre::FileInfoListPtr findFileInfo(const Ogre::String& pattern, bool recursive = true,
    bool dirs = false);

  /// @copydoc Archive::exists
  bool exists(const Ogre::String& filename);

private:
  void * mCurl;
  mutable void * mBuffer;
  std::string mLastURL;
  size_t mSize;
};

/** Specialisation of ArchiveFactory for URLArchive files. */
class _OgrePrivate URLArchiveFactory : public Ogre::ArchiveFactory
{
public:
  virtual ~URLArchiveFactory() {}
  /// @copydoc FactoryObj::getType
  const Ogre::String& getType(void) const;
  /// @copydoc FactoryObj::createInstance
  Ogre::Archive *createInstance( const Ogre::String& name ) 
  {
    return new URLArchive(name, "URL");
  }
  /// @copydoc FactoryObj::destroyInstance
  void destroyInstance( Ogre::Archive* arch) { delete arch; }
};

#endif

=== Add the new Archive Factory ===

In your Application.cpp file, after you've created the Root object but before calling setupResources(), add the following line:
<pre>
ArchiveManager::getSingleton().addArchiveFactory( new URLArchiveFactory() );
Of course, remember to include
URLArchive.h
here.

How to use it

To use, just add the URL base to the resources.cfg file like this:

[General]
URL=http://enno.homeunix.net/media/


And then use textures relative to the base URL in your materials:

material Examples/screenshot
{
  technique
  {
    pass
    {
      texture_unit
      {
        texture textures/brownsquares.jpg
      }
    }
  }
}

Improvements

The first thing to notice is that the exists() function is horribly slow. Because URLArchive::list() returns an empty vector, every resource gets checked and for each URLArchive you have specified in your resources, we try to download it from the server. The only concession to speed is that I'm saving the resulting buffer in case the next open() request is for the same URL (it usually is).

If you want to fix this, add a text file that contains all valid resources into the root of your web server (like a contents.txt file), read it at startup. You can use fetch_url() to get it, then parse the lines into a std::vector and either return false from exists if a requested file is not in the vector. This makes failures a lot faster because a missing resource doesn't have to ask the web server (and if you have several URL archives, you'll notice them).

Ordering the URL archives to the bottom of your resources.cfg file is a good idea, otherwise every resource would be tested against them first. Make sure the faster file system tests are done before you do the slow URL lookups.

The exists() call probably doesn't have to download the entire resource if you're a bit smarter about it, but as the next call is almost always an open() call anyhow, I saved myself the trouble.

You may want to add options to use a proxy server. I needed them, but cut them out for brevity.