BUG#: 8567

TITLE: The client side hang on when the server side fails to reallocate memory
DESCRIPTION: Server closes the connection and logs a message when out-of-memory error occurs during response encoding/aggregation or request handling.

diff --git a/src/Pegasus/Common/HTTPConnection.cpp b/src/Pegasus/Common/HTTPConnection.cpp
index c19cb34271285c6684b2c6d92aad1e2d4b901852..01cf262c716f2e281c6b317f1180db497dbdced2 100644 (file)
--- a/src/Pegasus/Common/HTTPConnection.cpp
+++ b/src/Pegasus/Common/HTTPConnection.cpp
@@ -130,6 +130,11 @@ static const Uint32 numberAsStringLength = 10;
 static const String httpDetailDelimiter = headerValueSeparator;
 static const String httpStatusInternal = HTTP_STATUS_INTERNALSERVERERROR;
 
+static const char INTERNAL_SERVER_ERROR_CONNECTION_CLOSED_KEY[] =
+    "Common.HTTPConnection.INTERNAL_SERVER_ERROR_CONNECTION_CLOSED";
+static const char INTERNAL_SERVER_ERROR_CONNECTION_CLOSED[] =
+    "Internal server error. Connection with IP address $0 closed.";
+
 /*
  * throw given http code with detail, file, line
  * This is shared client/server code. The caller will decide what to do
@@ -226,7 +231,8 @@ HTTPConnection::HTTPConnection(
     _contentLength(-1),
     _connectionClosePending(false),
     _acceptPending(false),
-    _firstRead(true)
+    _firstRead(true),
+    _internalError(false)
 {
     PEG_METHOD_ENTER(TRC_HTTP, "HTTPConnection::HTTPConnection");
 
@@ -290,6 +296,30 @@ void HTTPConnection::enqueue(Message *message)
     handleEnqueue(message);
 }
 
+void HTTPConnection::handleInternalServerError(
+    Uint32 respMsgIndex,
+    Boolean isComplete)
+{
+    PEG_METHOD_ENTER(TRC_HTTP, "HTTPConnection::handleInternalServerError");
+
+    PEG_TRACE((TRC_HTTP, Tracer::LEVEL1,
+        "Internal server error. Connection queue id : %u, IP address :%s, "
+            "Response Index :%u, Response is Complete :%u.",
+        getQueueId(),
+        (const char*)_ipAddress.getCString(),
+        respMsgIndex,
+        isComplete));
+
+    _internalError = true;
+    Buffer buffer;
+    HTTPMessage message(buffer);
+    message.setIndex(respMsgIndex);
+    message.setComplete(isComplete);
+    AutoMutex connectionLock(_connection_mut);
+    _handleWriteEvent(message); 
+    PEG_METHOD_EXIT();
+}
+
 void HTTPConnection::handleEnqueue(Message *message)
 {
     PEG_METHOD_ENTER(TRC_HTTP, "HTTPConnection::handleEnqueue");
@@ -440,6 +470,27 @@ Boolean HTTPConnection::_handleWriteEvent(HTTPMessage& httpMessage)
                 _transferEncodingChunkOffset++;
             }
 
+            // If there is an internal error on this connection, just return
+            // from here if the current message is not the last message becasue
+            // this connection will be closed once all messages are received.
+            if (_internalError)
+            {
+                if (isLast)
+                {
+                    _responsePending = false;
+                    _closeConnection();
+                    Logger::put_l(
+                        Logger::ERROR_LOG,
+                        System::CIMSERVER,
+                        Logger::SEVERE,
+                        MessageLoaderParms(
+                            INTERNAL_SERVER_ERROR_CONNECTION_CLOSED_KEY,
+                            INTERNAL_SERVER_ERROR_CONNECTION_CLOSED,
+                            _ipAddress));
+                }
+                return true;
+            }
+
             // save the first error
             if (httpMessage.cimException.getCode() != CIM_ERR_SUCCESS)
             {
@@ -974,11 +1025,25 @@ Boolean HTTPConnection::_handleWriteEvent(HTTPMessage& httpMessage)
     catch (Exception &e)
     {
         httpStatusString = e.getMessage();
+        _internalError = true;
+    }
+    catch (PEGASUS_STD(bad_alloc)&)
+    {
+        httpStatusString = "Out of memory";
+        _internalError = true;
     }
     catch (...)
     {
-        httpStatusString = HTTP_STATUS_INTERNALSERVERERROR;
-        PEG_TRACE_CSTRING(TRC_HTTP, Tracer::LEVEL1, "Unknown internal error");
+        httpStatusString = "Unknown error";
+        _internalError = true;
+    }
+
+    if (httpStatusString.size())
+    {
+        PEG_TRACE((TRC_HTTP, Tracer::LEVEL1,
+            "Internal error: %s, connection queue id: %u",
+            (const char*)httpStatusString.getCString(),
+            getQueueId()));
     }
 
     if (isLast == true)
@@ -1014,8 +1079,20 @@ Boolean HTTPConnection::_handleWriteEvent(HTTPMessage& httpMessage)
         //
         if (_isClient() == false)
         {
+            if (_internalError)
+            {
+                _closeConnection();
+                Logger::put_l(
+                    Logger::ERROR_LOG,
+                    System::CIMSERVER,
+                    Logger::SEVERE,
+                    MessageLoaderParms(
+                        INTERNAL_SERVER_ERROR_CONNECTION_CLOSED_KEY,
+                        INTERNAL_SERVER_ERROR_CONNECTION_CLOSED,
+                        _ipAddress));
+            }
             // Check for message to close
-            if (httpMessage.getCloseConnect())
+            else if (httpMessage.getCloseConnect())
             {
                 PEG_TRACE((TRC_HTTP, Tracer::LEVEL3,
                     "HTTPConnection::_handleWriteEvent: \"Connection: Close\" "

diff --git a/src/Pegasus/Common/HTTPConnection.h b/src/Pegasus/Common/HTTPConnection.h
index e60f16d5d0afd811be6fd85b1662cd5fe7708b89..b6550dbfc98cebd0b093676564790370022d5ad4 100644 (file)
--- a/src/Pegasus/Common/HTTPConnection.h
+++ b/src/Pegasus/Common/HTTPConnection.h
@@ -104,6 +104,16 @@ public:
 
     Boolean closeConnectionOnTimeout(struct timeval* timeNow);
 
+    // This method is called in Server code when response encoders or
+    // HTTPAuthenticatorDelegator runs out-of-memory. This method calls 
+    // _handleWriteEvent() with a dummy HTTPMessage to maintain  response
+    // chunk sequence properly. Once all responses are  arrived, connection
+    // is closed. Param "respMsgIndex" indicates the response index and 
+    // isComplete indicates whether the response is complete or not.
+    void handleInternalServerError(
+        Uint32 respMsgIndex,
+        Boolean isComplete);
+
     // ATTN-RK-P1-20020521: This is a major hack, required to get the CIM
     // server and tests to run successfully.  The problem is that the
     // HTTPAcceptor is deleting an HTTPConnection before all the threads
@@ -221,6 +231,11 @@ private:
 #ifndef PEGASUS_INTEGERS_BOUNDARY_ALIGNED
     static Mutex _idleConnectionTimeoutSecondsMutex;
 #endif
+    // When this flag is set to true, it indicates that internal error on this
+    // connection occured. Currently this flag is used by the Server code when
+    // out-of-memory error is occurs and connection is closed by the server
+    // once all responses are arrived.
+    Boolean _internalError;
 
     friend class Monitor;
     friend class HTTPAcceptor;

diff --git a/src/Pegasus/Server/CIMOperationResponseEncoder.cpp b/src/Pegasus/Server/CIMOperationResponseEncoder.cpp
index 7ab733f41281b481b414ee6ea35ab1599f4605fb..2b40d83004222229c7f288e1786e84c940433955 100644 (file)
--- a/src/Pegasus/Server/CIMOperationResponseEncoder.cpp
+++ b/src/Pegasus/Server/CIMOperationResponseEncoder.cpp
@@ -125,7 +125,6 @@ void CIMOperationResponseEncoder::sendResponse(
     }
 
     Uint32 queueId = response->queueIds.top();
-    response->queueIds.pop();
 
     Boolean closeConnect = response->getCloseConnect();
     PEG_TRACE((
@@ -264,38 +263,15 @@ void CIMOperationResponseEncoder::sendResponse(
     }
     else
     {
-        // else non-error condition
-        try
-        {
-            message = formatResponse(
-                cimName,
-                messageId,
-                httpMethod,
-                contentLanguage,
-                body,
-                serverTime,
-                isFirst,
-                isLast);
-        }
-        catch (PEGASUS_STD(bad_alloc)&)
-        {
-            MessageLoaderParms parms(
-                "Server.CIMOperationResponseEncoder.OUT_OF_MEMORY",
-                "A System error has occurred. Please retry the CIM Operation "
-                    "at a later time.");
-
-            Logger::put_l(
-                Logger::ERROR_LOG, System::CIMSERVER, Logger::WARNING,
-                parms);
-
-            cimException = PEGASUS_CIM_EXCEPTION_L(CIM_ERR_FAILED, parms);
-
-            // try again with new error and no body
-            body.clear();
-            sendResponse(response, name, isImplicit);
-            PEG_METHOD_EXIT();
-            return;
-        }
+        message = formatResponse(
+            cimName,
+            messageId,
+            httpMethod,
+            contentLanguage,
+            body,
+            serverTime,
+            isFirst,
+            isLast);
 
         STAT_BYTESSENT
     }
@@ -328,7 +304,33 @@ void CIMOperationResponseEncoder::sendResponse(
 
 void CIMOperationResponseEncoder::enqueue(Message* message)
 {
-    handleEnqueue(message);
+    try
+    {
+        handleEnqueue(message);
+    }
+    catch(PEGASUS_STD(bad_alloc)&)
+    {
+        MessageLoaderParms parms(
+            "Server.CIMOperationResponseEncoder.OUT_OF_MEMORY",
+            "A System error has occurred. Please retry the CIM Operation "
+                "at a later time.");
+
+        Logger::put_l(
+            Logger::ERROR_LOG, System::CIMSERVER, Logger::SEVERE, parms);
+
+        CIMResponseMessage* response =
+            dynamic_cast<CIMResponseMessage*>(message);
+        Uint32 queueId = response->queueIds.top();
+        MessageQueue* queue = MessageQueue::lookup(queueId);
+        HTTPConnection* httpQueue = dynamic_cast<HTTPConnection*>(queue);
+        PEGASUS_ASSERT(httpQueue);
+
+        // Handle internal error on this connection.
+        httpQueue->handleInternalServerError(
+            response->getIndex(), response->isComplete());
+
+        delete message;
+    }
 }
 
 void CIMOperationResponseEncoder::handleEnqueue(Message* message)

diff --git a/src/Pegasus/Server/HTTPAuthenticatorDelegator.cpp b/src/Pegasus/Server/HTTPAuthenticatorDelegator.cpp
index 164d7a389ca641ec072859cd212eb0a997f589d2..43f553d16f02555b67fc6528744722742afcd91a 100644 (file)
--- a/src/Pegasus/Server/HTTPAuthenticatorDelegator.cpp
+++ b/src/Pegasus/Server/HTTPAuthenticatorDelegator.cpp
@@ -1221,12 +1221,13 @@ void HTTPAuthenticatorDelegator::handleHTTPMessage(
                 catch (const bad_alloc&)
                 {
                     delete httpMessage;
-                    _sendHttpError(
-                        queueId,
-                        HTTP_STATUS_REQUEST_TOO_LARGE,
-                        String::EMPTY,
-                        String::EMPTY,
-                        closeConnect);
+                    HTTPConnection *httpQueue =
+                        dynamic_cast<HTTPConnection*>(
+                             MessageQueue::lookup(queueId));
+                    if (httpQueue)
+                    {
+                        httpQueue->handleInternalServerError(0, true);
+                    }
                     PEG_METHOD_EXIT();
                     deleteMessage = false;
                     return;
@@ -1284,12 +1285,13 @@ void HTTPAuthenticatorDelegator::handleHTTPMessage(
                 catch (const bad_alloc&)
                 {
                     delete httpMessage;
-                    _sendHttpError(
-                       queueId,
-                       HTTP_STATUS_REQUEST_TOO_LARGE,
-                       String::EMPTY,
-                       String::EMPTY,
-                       closeConnect);
+                    HTTPConnection *httpQueue =
+                        dynamic_cast<HTTPConnection*>(
+                             MessageQueue::lookup(queueId));
+                    if (httpQueue)
+                    {
+                        httpQueue->handleInternalServerError(0, true);
+                    }
                     PEG_METHOD_EXIT();
                     deleteMessage = false;
                     return;

diff --git a/src/Pegasus/WsmServer/WsmResponseEncoder.cpp b/src/Pegasus/WsmServer/WsmResponseEncoder.cpp
index 951fa1bc6454033c9a77c2ab4fe965550ab63339..2342365e83e06894dea70ae99cd7d865d69c2667 100644 (file)
--- a/src/Pegasus/WsmServer/WsmResponseEncoder.cpp
+++ b/src/Pegasus/WsmServer/WsmResponseEncoder.cpp
@@ -206,18 +206,20 @@ void WsmResponseEncoder::enqueue(WsmResponse* response)
     }
     catch (PEGASUS_STD(bad_alloc)&)
     {
-        WsmFault fault(WsmFault::wsman_InternalError,
-            MessageLoaderParms(
-                "WsmServer.WsmResponseEncoder.OUT_OF_MEMORY",
-                "A System error has occurred. Please retry the "
-                    "WS-Management operation at a later time."));
-        WsmFaultResponse outofmem(
-            response->getRelatesTo(),
-            response->getQueueId(),
-            response->getHttpMethod(),
-            response->getHttpCloseConnect(),
-            fault);
-        _encodeWsmFaultResponse(&outofmem);
+       MessageLoaderParms parms(
+            "WsmServer.WsmResponseEncoder.OUT_OF_MEMORY",
+            "A System error has occurred. Please retry the "
+                "WS-Management operation at a later time.");
+
+        Logger::put_l(
+            Logger::ERROR_LOG, System::CIMSERVER, Logger::SEVERE, parms);
+        
+        MessageQueue* queue = MessageQueue::lookup(response->getQueueId());
+        HTTPConnection* httpQueue = dynamic_cast<HTTPConnection*>(queue);
+        PEGASUS_ASSERT(httpQueue);
+
+        // Handle the internal server error on this connection.
+        httpQueue->handleInternalServerError(0, true);
     }
 
     PEG_METHOD_EXIT();

diff --git a/src/Pegasus/msg/Server/pegasusServer_en.txt b/src/Pegasus/msg/Server/pegasusServer_en.txt
index 3adbb6281911476d2768f8ee2f5071abef373f53..1bafe29b49d6f27df88ac8e50a46b52202612925 100644 (file)
--- a/src/Pegasus/msg/Server/pegasusServer_en.txt
+++ b/src/Pegasus/msg/Server/pegasusServer_en.txt
@@ -2864,6 +2864,11 @@ en:table {
 
         Common.HTTPConnection.UNKNOWN_EXCEPTION:string {"PGS08100: Unknown exception caught while parsing HTTP message."}
 
+        /**
+        * @note  PGS08101:
+        *    Substitution {0} is a string containing the host of the CIMClient connection
+        */
+        Common.HTTPConnection.INTERNAL_SERVER_ERROR_CONNECTION_CLOSED:string {"PGS08101: Internal server error. Connection with IP address {0} closed."}
 
         // ==========================================================
         // Messages for LanguageParser